WIP: template now creates netCDF file

* added method to automatically update dimensions from values arrays
* added writing of variable values to file
* create() method now does everything (don't need to set outfile first)
* removed ability to specify attribute types in JSON for now

Author: mhidas

ncwriter/__init__.py

@@ -2,4 +2,4 @@
 
 __all__ = [
     'DatasetTemplate'
-]
+]

ncwriter/template.py

@@ -11,6 +11,7 @@
 # TODO check_var too complex
 # TODO createVariables too complex
 # TODO implement from_file/from_cdl/from_json kwarg!?
+# TODO Allow for attribute types to be specified in JSON
 
 import json
 from collections import OrderedDict
@@ -273,11 +274,6 @@ def from_json(cls, path):
                    global_attributes=template.get('global_attributes')
                    )
 
-    def set_output(self, outfile, mode='w', **kwargs):
-        """Create the dataset """
-        self.outfile = outfile
-        self.ncobj = netCDF4.Dataset(self.outfile, mode=mode, **kwargs)
-
     def _create_var_opts(self, vdict):
         """Return a list with attribute names required for the creation of variable
         defined by :vdict: This include creation/special options like:
@@ -295,6 +291,40 @@ def _create_var_opts(self, vdict):
             inside = inside.union(aliases)
         return list(inside)
 
+    def update_dimensinos(self):
+        """Update the sizes of dimensions to be consistent with the arrays set as variable values, if possible.
+        Otherwise raise ValueError. Also raise ValueError if a dimension that already has a non-zero size is not
+        consistent with variable array sizes.
+        """
+        for name, var in self.variables.iteritems():
+            values = var.get('values')
+            if values is None:
+                continue
+
+            var_shape = values.shape
+            var_dims = var.get('dims', [])
+            if len(var_shape) != len(var_dims):
+                raise ValueError(
+                    "Variable '{name}' has {ndim} dimensions, but value array has {nshape} dimensions.".format(
+                        name=name, ndim=len(var_dims), nshape=len(var_shape)
+                    )
+                )
+
+            for dim, size in zip(var_dims, var_shape):
+                template_dim = self.dimensions[dim]
+                if template_dim is None or template_dim == 0:
+                    self.dimensions[dim] = size
+
+            # check that shape is now consistent
+            template_shape = tuple(self.dimensions[d] for d in var_dims)
+            if var_shape != template_shape:
+                raise ValueError(
+                    "Variable '{name}' has dimensions {var_dims} and shape {var_shape}, inconsistent with dimension "
+                    "sizes defined in template {template_shape}".format(
+                        name=name, var_dims=var_dims, var_shape=var_shape, template_shape=template_shape
+                    )
+                )
+
     def createDimensions(self):
         """Create the dimensions on the netcdf file"""
         for dname, dval in zip(self.dimensions.keys(),
@@ -306,20 +336,17 @@ def createVariables(self, **kwargs):
         **kwargs are included here to overload all options for all variables
         like `zlib` and friends.
         """
-        for v in self.variables.keys():
-            varname = v #self.variables[v]['name']
-            datatype = self.variables[v]['type']
-            dimensions = self.variables[v]['dims']
-
-            var_c_opts = {}
+        for varname, var in self.variables.iteritems():
+            datatype = var['type']
+            dimensions = var['dims']
             cwargs = kwargs.copy()
             if dimensions is None:  # no kwargs in createVariable
-                self.ncobj.createVariable(varname, datatype)
+                ncvar = self.ncobj.createVariable(varname, datatype)
             else:
-                var_c_keys = list(self._create_var_opts(self.variables[v]))
+                var_c_keys = list(self._create_var_opts(var))
 
                 var_c_opts = dict(
-                    (x, self.variables[v][x]) for x in var_c_keys)
+                    (x, var[x]) for x in var_c_keys)
 
                 ureq_fillvalue = [
                     x for x in cwargs.keys() if x in self.fill_aliases
@@ -344,32 +371,33 @@ def createVariables(self, **kwargs):
                     if fv_val:
                         var_c_opts['fill_value'] = fv_val[-1]
 
-                self.ncobj.createVariable(
+                ncvar = self.ncobj.createVariable(
                     varname, datatype, dimensions=dimensions, **var_c_opts)
 
-            if 'attr' in self.variables[v].keys():
-                attrs = self.variables[v]['attr'].copy()
+            # add variable values
+            ncvar[:] = var['values']
+
+            # add variable attributes
+            if var.get('attr'):
+                attrs = var['attr'].copy()
                 for not_attr in self._create_var_opts(attrs):
                     attrs.pop(not_attr)
-
-                for attname in attrs.keys():
-                    var = self.ncobj.variables[varname]
-                    value = np.array(attrs[attname]['value']).astype(
-                        attrs[attname]['type'])
-                    var.setncattr(attname, value)
+                ncvar.setncatts(attrs)
 
     def createGlobalAttrs(self):
         """Add the global attributes for the current class"""
         for att in self.global_attributes.keys():
             self.ncobj.setncattr(att, self.global_attributes[att])
 
-    def create(self, **kwargs):
-        """Create in the dimensions/variable and attributes and fill with
-        basic information"""
+    def create(self, outfile, mode='w', var_args={}, **kwargs):
+        """Create the file according to all the information in the template"""
+        self.outfile = outfile
+        self.ncobj = netCDF4.Dataset(self.outfile, mode=mode, **kwargs)
+
+        self.update_dimensinos()
         self.createDimensions()
-        self.createVariables(**kwargs)
+        self.createVariables(**var_args)
         self.createGlobalAttrs()
         self.ncobj.sync()
         self.ncobj.close()
         self.ncobj = netCDF4.Dataset(self.outfile, 'a')
-        pass

test_ncwriter/template1.json

@@ -26,13 +26,18 @@
             "type": "float32",
             "attr": {
                 "standard_name": "sea_water_temperature",
-                "units": "degC"
+                "units": "degC",
+                "valid_min": 0.0,
+                "valid_max": 42.00
             }
         }
     },
     "global_attributes": {
         "title": "test dataset",
         "abstract": "This is a dataset used for testing",
-        "Conventions": "CF-1.6,IMOS-1.4"
+        "Conventions": "CF-1.6,IMOS-1.4",
+        "number": 10,
+        "big_number": 12345678901234567890,
+        "big_float": 1.234567890123456e777
     }
 }

test_ncwriter/test_template.py

@@ -128,21 +128,34 @@ def test_create_file(self):
         template = DatasetTemplate.from_json(TEMPLATE_JSON)
         template.variables['TIME']['values'] = self.values10
         template.variables['DEPTH']['values'] = self.values1
-        template.variables['TEMP']['values'] = self.values10
+        template.variables['TEMP']['values'] = self.values10.reshape((10, 1))
         template.create(self.temp_nc_file)
+        print("Created nc file '{}'".format(self.temp_nc_file))
 
         dataset = Dataset(self.temp_nc_file)
 
+        expected_dimensions = OrderedDict([
+            ('TIME', len(self.values10)),
+            ('DEPTH', len(self.values1))
+        ])
         ds_dimensions = OrderedDict((k, v.size) for k, v in dataset.dimensions.iteritems())
-        self.assertEqual(self.dimensions, ds_dimensions)
+        self.assertEqual(expected_dimensions, ds_dimensions)
 
-        for vname, vdict in self.variables:
+        for vname, vdict in self.variables.iteritems():
             ds_var = dataset[vname]
-            self.assertEqual(vdict['dims'], ds_var.dimensions)
-            ds_var_attr = OrderedDict((k, ds_var[k]) for k in ds_var.ncattrs())
-            self.assertEqual(vdict['attr'], ds_var_attr)
-
-        ds_global_attributes = OrderedDict((k, dataset[k]) for k in dataset.ncattrs())
+            self.assertEqual(vdict['dims'], list(ds_var.dimensions))
+            self.assertEqual(vdict['type'], ds_var.dtype)
+            ds_var_attr = OrderedDict((k, ds_var.getncattr(k)) for k in ds_var.ncattrs())
+            if vdict['attr'] is None:
+                self.assertEqual({}, ds_var_attr)
+            else:
+                self.assertEqual(vdict['attr'], ds_var_attr)
+
+        self.assertTrue(all(dataset['TIME'] == self.values10))
+        self.assertTrue(all(dataset['DEPTH'] == self.values1))
+        self.assertTrue(all(dataset['TEMP'] == self.values10.reshape(10, 1)))
+
+        ds_global_attributes = OrderedDict((k, dataset.getncattr(k)) for k in dataset.ncattrs())
         self.assertEqual(self.global_attributes, ds_global_attributes)
 
 # TODO: add data from multiple numpy arrays