Merge pull request #172 from aodn/fix_function_arg_modification

Fix functions that modify files_to_agg argument in place

Author: alexgleith

aodntools/timeseries_products/aggregated_timeseries.py

@@ -1,6 +1,7 @@
 #!/usr/bin/env python3
 
 import argparse
+from copy import deepcopy
 import json
 import os
 import shutil
@@ -225,6 +226,7 @@ def main_aggregator(files_to_agg, var_to_agg, site_code, input_dir='', output_di
                 rejected_files.append(file)
 
     ## remove bad files form the list and sort in chronological order
+    files_to_agg = deepcopy(files_to_agg)
     for file in bad_files.keys():
         files_to_agg.remove(file)
     if len(files_to_agg) == 0:

aodntools/timeseries_products/velocity_aggregated_timeseries.py

@@ -1,6 +1,8 @@
 import os
 import tempfile
 import shutil
+from copy import deepcopy
+
 from  netCDF4 import Dataset, num2date, stringtochar
 import numpy as np
 import json
@@ -83,6 +85,7 @@ def velocity_aggregated(files_to_agg, site_code, input_dir='', output_dir='./',
                 bad_files.update({file: error_list})
 
     # remove bad files form the list and sort in chronological order
+    files_to_agg = deepcopy(files_to_agg)
     for file in bad_files.keys():
         files_to_agg.remove(file)
     if len(files_to_agg) == 0:

aodntools/timeseries_products/velocity_hourly_timeseries.py

@@ -3,6 +3,7 @@
 import os
 import shutil
 import tempfile
+from copy import deepcopy
 
 import numpy as np
 import pandas as pd
@@ -117,6 +118,7 @@ def velocity_hourly_aggregated(files_to_agg, site_code, input_dir='', output_dir
     print(" ")
 
     ## remove bad files form the list
+    files_to_agg = deepcopy(files_to_agg)
     for file in bad_files.keys():
         files_to_agg.remove(file)
     if len(files_to_agg) == 0:

test_aodntools/timeseries_products/test_aggregated_timeseries.py

@@ -29,6 +29,7 @@ def test_main_aggregator(self):
         output_file, bad_files = main_aggregator(INPUT_FILES, 'TEMP', 'NRSROT', input_dir=TEST_ROOT,
                                                  output_dir='/tmp')
 
+        self.assertEqual(4, len(INPUT_FILES))
         self.assertEqual(1, len(bad_files))
         for file, errors in bad_files.items():
             self.assertEqual(BAD_FILE, file)

test_aodntools/timeseries_products/test_velocity_aggregated_timeseries.py

@@ -33,6 +33,7 @@ class TestVelocityAggregatedTimeseries(BaseTestCase):
     def test_velocity_aggregated(self):
         output_file, bad_files = velocity_aggregated(INPUT_FILES, 'NRSROT', input_dir=TEST_ROOT, output_dir='/tmp')
 
+        self.assertEqual(4, len(INPUT_FILES))
         self.assertEqual(1, len(bad_files))
         for file, errors in bad_files.items():
             self.assertEqual(BAD_FILE, file)

test_aodntools/timeseries_products/test_velocity_hourly_timeseries.py

@@ -37,7 +37,7 @@ class TestVelocityHourlyTimeseries(BaseTestCase):
     def test_velocity_hourly(self):
         output_file, bad_files = velocity_hourly_aggregated(INPUT_FILES, 'NRSROT',
                                                             input_dir=TEST_ROOT, output_dir='/tmp')
-
+        self.assertEqual(4, len(INPUT_FILES))
         self.assertEqual(1, len(bad_files))
         for file, errors in bad_files.items():
             self.assertEqual(BAD_FILE, file)