Merge pull request #158 from aodn/fix_velocity_time_scrambing

Fix time scrambing issue in velocity hourly product

Author: leonardolaiolo

aodntools/timeseries_products/velocity_hourly_timeseries.py

@@ -57,13 +57,12 @@ def append_resampled_values(nc_cell, ds, slice_start, binning_functions):
     df_cell = nc_cell.squeeze().to_dataframe()
     # shift the index forward 30min to centre the bins on the hour
     df_cell.index = df_cell.index + pd.Timedelta(minutes=30)
-    # TODO: shift timestamps to centre of sampling interval
 
     df_cell_1H = df_cell.resample('1H')
     slice_end = len(df_cell_1H) + slice_start
 
     # set binned timestamps
-    time_slice = (np.fromiter(df_cell_1H.groups.keys(), dtype='M8[ns]') - TIME_EPOCH) / ONE_DAY
+    time_slice = (np.fromiter(df_cell_1H.min().index, dtype='M8[ns]') - TIME_EPOCH) / ONE_DAY
     ds['TIME'][slice_start:slice_end] = time_slice
 
     # take the mean of the variables

test_aodntools/timeseries_products/IMOS_ANMN-NRS_VZ_20180816_NRSROT_FV02_velocity-hourly-timeseries_END-20191018_C-20220502.nc

@@ -22,8 +22,14 @@
     TEST_ROOT, 'IMOS_ANMN-NRS_VZ_20180816_NRSROT_FV01_velocity-aggregated-timeseries_END-20191018_C-20200623.nc'
 )
 
+OBS_VARS = {'TIME', 'DEPTH', 'DEPTH_quality_control', 'UCUR', 'UCUR_quality_control',
+            'VCUR', 'VCUR_quality_control', 'WCUR', 'WCUR_quality_control', 'instrument_index', 'CELL_INDEX'}
+INST_VARS = {'LATITUDE', 'LONGITUDE', 'NOMINAL_DEPTH', 'SECONDS_TO_MIDDLE'}
+STR_VARS = {'source_file', 'instrument_id'}
+
+
 class TestVelocityAggregatedTimeseries(BaseTestCase):
-    def test_main_aggregator(self):
+    def test_velocity_aggregated(self):
         output_file, bad_files = velocity_aggregated(INPUT_FILES, 'NRSROT', input_dir=TEST_ROOT, output_dir='/tmp')
 
         self.assertEqual(1, len(bad_files))
@@ -34,19 +40,13 @@ def test_main_aggregator(self):
 
         # check dimensions and variables
         self.assertSetEqual(set(dataset.dimensions), {'OBSERVATION', 'INSTRUMENT', 'strlen'})
+        self.assertSetEqual(set(dataset.variables.keys()), OBS_VARS | INST_VARS | STR_VARS)
 
-        obs_vars = {'TIME', 'DEPTH', 'DEPTH_quality_control', 'UCUR', 'UCUR_quality_control',
-                    'VCUR', 'VCUR_quality_control', 'WCUR', 'WCUR_quality_control', 'instrument_index', 'CELL_INDEX'}
-        inst_vars = {'LATITUDE', 'LONGITUDE', 'NOMINAL_DEPTH', 'SECONDS_TO_MIDDLE'}
-        string_vars = {'source_file', 'instrument_id'}
-
-        self.assertSetEqual(set(dataset.variables.keys()), obs_vars | inst_vars | string_vars)
-
-        for var in obs_vars:
+        for var in OBS_VARS:
             self.assertEqual(dataset.variables[var].dimensions, ('OBSERVATION',))
-        for var in inst_vars:
+        for var in INST_VARS:
             self.assertEqual(dataset.variables[var].dimensions, ('INSTRUMENT',))
-        for var in string_vars:
+        for var in STR_VARS:
             self.assertEqual(dataset.variables[var].dimensions, ('INSTRUMENT', 'strlen'))
 
         for f in chartostring(dataset['source_file'][:]):
@@ -58,8 +58,7 @@ def test_main_aggregator(self):
 
         # check aggregated variable values
         expected = Dataset(EXPECTED_OUTPUT_FILE)
-        compare_vars = ('TIME', 'UCUR', 'UCUR_quality_control', 'VCUR', 'VCUR_quality_control', 'NOMINAL_DEPTH',
-                        'CELL_INDEX', 'instrument_index', 'DEPTH', 'DEPTH_quality_control')
+        compare_vars = set(expected.variables.keys()) - STR_VARS
         non_match_vars = [var for var in compare_vars
                           if not all(dataset[var][:] == expected[var][:])
                           ]

test_aodntools/timeseries_products/test_velocity_aggregated_timeseries.py

@@ -0,0 +1,80 @@
+#!/usr/bin/env python3
+
+import os
+import unittest
+
+import numpy as np
+from netCDF4 import Dataset, chartostring
+
+from aodntools import __version__
+from aodntools.timeseries_products.common import NoInputFilesError
+from aodntools.timeseries_products.velocity_hourly_timeseries import velocity_hourly_aggregated
+from test_aodntools.base_test import BaseTestCase
+
+TEST_ROOT = os.path.dirname(__file__)
+BAD_FILE = 'IMOS_ANMN-NRS_BAD_VELOCITY_FILE.nc'
+INPUT_FILES = [
+    'IMOS_ANMN-NRS_AETVZ_20181213T080000Z_NRSROT-ADCP_FV01_NRSROT-ADCP-1812-Sentinel-or-Monitor-Workhorse-ADCP-44_END-20181215T100000Z_C-20200430T000000Z.nc',
+    'IMOS_ANMN-NRS_AETVZ_20180816T080000Z_NRSROT-ADCP_FV01_NRSROT-ADCP-1808-Sentinel-or-Monitor-Workhorse-ADCP-44_END-20180822T053000Z_C-20200623T000000Z.nc',
+    'IMOS_ANMN-NRS_AETVZ_20191016T080000Z_NRSROT-ADCP_FV01_NRSROT-ADCP-1910-Sentinel-or-Monitor-Workhorse-ADCP-44_END-20191018T100000Z_C-20200430T000000Z.nc',
+    BAD_FILE
+]
+EXPECTED_OUTPUT_FILE = os.path.join(
+    TEST_ROOT, 'IMOS_ANMN-NRS_VZ_20180816_NRSROT_FV02_velocity-hourly-timeseries_END-20191018_C-20220502.nc'
+)
+
+OBS_VARS = {'TIME', 'instrument_index', 'CELL_INDEX'}
+INST_VARS = {'LATITUDE', 'LONGITUDE', 'NOMINAL_DEPTH', 'SECONDS_TO_MIDDLE'}
+STR_VARS = {'source_file', 'instrument_id'}
+for v in ['DEPTH', 'UCUR', 'VCUR', 'WCUR']:
+    OBS_VARS.add(v)
+    for s in ['_min', '_max', '_std', '_count']:
+        OBS_VARS.add(v + s)
+
+
+class TestVelocityHourlyTimeseries(BaseTestCase):
+    def test_velocity_hourly(self):
+        output_file, bad_files = velocity_hourly_aggregated(INPUT_FILES, 'NRSROT',
+                                                            input_dir=TEST_ROOT, output_dir='/tmp')
+
+        self.assertEqual(1, len(bad_files))
+        for file, errors in bad_files.items():
+            self.assertEqual(BAD_FILE, file)
+
+        dataset = Dataset(output_file)
+
+        # check dimensions and variables
+        self.assertSetEqual(set(dataset.dimensions), {'OBSERVATION', 'INSTRUMENT', 'strlen'})
+        self.assertSetEqual(set(dataset.variables.keys()), OBS_VARS | INST_VARS | STR_VARS)
+
+        for var in OBS_VARS:
+            self.assertEqual(dataset.variables[var].dimensions, ('OBSERVATION',))
+        for var in INST_VARS:
+            self.assertEqual(dataset.variables[var].dimensions, ('INSTRUMENT',))
+        for var in STR_VARS:
+            self.assertEqual(dataset.variables[var].dimensions, ('INSTRUMENT', 'strlen'))
+
+        for f in chartostring(dataset['source_file'][:]):
+            self.assertIn(f, INPUT_FILES)
+
+        # check attributes
+        self.assertEqual(__version__, dataset.generating_code_version)
+        self.assertIn(__version__, dataset.lineage)
+
+        # check aggregated variable values
+        expected = Dataset(EXPECTED_OUTPUT_FILE)
+        self.assertEqual(len(expected['TIME']), len(dataset['TIME']))
+
+        non_match_vars = []
+        for var in set(expected.variables.keys()) - STR_VARS:
+            if not all(np.isclose(dataset[var], expected[var], equal_nan=True)):
+                non_match_vars.append(var)
+        self.assertEqual(non_match_vars, [])
+
+    def test_all_rejected(self):
+        self.assertRaises(NoInputFilesError, velocity_hourly_aggregated, [BAD_FILE], 'NRSROT',
+                          input_dir=TEST_ROOT, output_dir='/tmp')
+
+
+if __name__ == '__main__':
+    unittest.main()