Add method for computing statistics about vector layers

Author: doruk.ozturk

dataqs/helpers.py

@@ -31,7 +31,7 @@
 import sys
 from StringIO import StringIO
 import rasterio
-from osgeo import gdal
+from osgeo import gdal, ogr
 from rasterio.warp import RESAMPLING
 from rasterio.warp import calculate_default_transform, reproject
 import unicodedata
@@ -390,6 +390,50 @@ def add_keywords(keyword_list, extra_keywords):
     # keyword list
     filtered_keywords = [k for k in keyword_list if not
                          (k.startswith('category:') or
-                          k.startswith('datetime:'))]
+                          k.startswith('datetime:') or
+                          k.startswith('layer_info'))]
 
     return filtered_keywords + extra_keywords
+
+
+def _getMinMax(layer, field):
+    fieldVal = [f.GetField(field) for f in layer]
+    return {'properties': {'min': min(fieldVal), 'max': max(fieldVal)},
+            'type': 'numeric'}
+
+
+def _getNumericFields(layer):
+    """ Gets only the numeric fields from layer"""
+
+    layerDefinition = layer.GetLayerDefn()
+    numFields = []
+    for i in range(layerDefinition.GetFieldCount()):
+        fieldName = layerDefinition.GetFieldDefn(i).GetName()
+        fieldTypeCode = layerDefinition.GetFieldDefn(i).GetType()
+        fieldDef = layerDefinition.GetFieldDefn(i)
+        fieldType = fieldDef.GetFieldTypeName(fieldTypeCode)
+        if fieldType != 'String':
+            numFields.append(fieldName)
+
+    return numFields
+
+
+def get_vector_layer_info(geojson):
+    """ Gets information about a given geojson file """
+
+    dataSource = ogr.Open(geojson)
+    layer = dataSource.GetLayer()
+    geom = {1: 'point', 2: 'line', 3: 'polygon'}
+    subType = geom[layer.GetGeomType()]
+    count = layer.GetFeatureCount()
+    numFields = _getNumericFields(layer)
+    info = {'layerType': 'vector', 'subType': subType}
+    attr = {}
+    for f in numFields:
+        prop = _getMinMax(layer, f)
+        prop['properties']['count'] = count
+        attr[f.lower()] = prop
+        layer.ResetReading()
+    dataSource = None
+    info['attributes'] = attr
+    return info

dataqs/hifld/hifld.py

@@ -18,13 +18,15 @@
 ###############################################################################
 
 from __future__ import absolute_import
+import json
 import os
 import logging
 import time
 import traceback
 from django.conf import settings
 from dataqs.processor_base import GeoDataProcessor, DEFAULT_WORKSPACE
-from dataqs.helpers import ogr2ogr_exec, layer_exists, style_exists
+from dataqs.helpers import ogr2ogr_exec, layer_exists, style_exists,\
+    get_vector_layer_info
 from geonode.geoserver.helpers import ogc_server_settings
 
 logger = logging.getLogger("dataqs.processors")
@@ -80,6 +82,8 @@ def run(self):
                         lyr_file = os.path.join(
                             self.tmp_dir,
                             self.download(layer['url'], filename=table))
+                info = get_vector_layer_info(os.path.join(self.tmp_dir, table))
+                layer_info = 'layer_info:{}'.format(json.dumps(info))
                 db = ogc_server_settings.datastore_db
                 ogr2ogr_exec("-overwrite -skipfailures -f PostgreSQL \
                     \"PG:host={db_host} user={db_user} password={db_pass} \
@@ -106,7 +110,7 @@ def run(self):
                     title=layer['name'],
                     description=layer['description'],
                     store=datastore,
-                    extra_keywords=[keywords])
+                    extra_keywords=[keywords, layer_info])
                 self.truncate_gs_cache(table)
             except Exception:
                 logger.error('Error with layer {}'.format(layer['name']))