Merge pull request #50 from OpenGeoscience/precompute-metadata

Precompute metadata

Author: dorukozturk

ansible/Vagrantfile

@@ -12,7 +12,7 @@ Vagrant.configure("2") do |config|
     # Ref: https://github.com/Kitware/minerva/pull/98#issuecomment-139023062
     config.vm.network "private_network", ip: "192.168.33.12"
     geoservices.vm.provider :virtualbox do |vb|
-       vb.customize [ "modifyvm", :id, "--name", "geoservices","--memory", 6144 ]
+       vb.customize [ "modifyvm", :id, "--name", "geoservices","--memory", 10000 ]
         end
     geoservices.vm.provision "ansible" do |ansible|
       ansible.playbook = "playbook.yml"

dataqs/helpers.py

@@ -31,7 +31,7 @@
 import sys
 from StringIO import StringIO
 import rasterio
-from osgeo import gdal
+from osgeo import gdal, ogr
 from rasterio.warp import RESAMPLING
 from rasterio.warp import calculate_default_transform, reproject
 import unicodedata
@@ -390,6 +390,60 @@ def add_keywords(keyword_list, extra_keywords):
     # keyword list
     filtered_keywords = [k for k in keyword_list if not
                          (k.startswith('category:') or
-                          k.startswith('datetime:'))]
+                          k.startswith('datetime:') or
+                          k.startswith('layer_info'))]
 
     return filtered_keywords + extra_keywords
+
+
+def _getMinMax(layer, field):
+    min_value, max_value = float('inf'), float('-inf')
+    for f in layer:
+        value = f.GetField(field)
+        if value < min_value:
+            min_value = value
+        if value > max_value:
+            max_value = value
+
+    return {'properties': {'min': min_value, 'max': max_value},
+            'type': 'numeric'}
+
+
+def _getNumericFields(layer):
+    """ Gets only the numeric fields from layer"""
+
+    layerDefinition = layer.GetLayerDefn()
+    numFields = []
+    for i in xrange(layerDefinition.GetFieldCount()):
+        fieldName = layerDefinition.GetFieldDefn(i).GetName()
+        fieldTypeCode = layerDefinition.GetFieldDefn(i).GetType()
+        fieldDef = layerDefinition.GetFieldDefn(i)
+        fieldType = fieldDef.GetFieldTypeName(fieldTypeCode)
+        if fieldType != 'String':
+            numFields.append(fieldName)
+
+    return numFields
+
+
+def get_vector_layer_info(geojson):
+    """ Gets information about a given geojson file """
+
+    dataSource = ogr.Open(geojson)
+    layer = dataSource.GetLayer()
+    geom = {0: 'polygon', 1: 'point', 2: 'line',
+            3: 'polygon', 4: 'polygon', 5: 'polygon',
+            6: 'polygon', -2147483647: 'point'}
+
+    subType = geom[layer.GetGeomType()]
+    count = layer.GetFeatureCount()
+    numFields = _getNumericFields(layer)
+    info = {'layerType': 'vector', 'subType': subType}
+    attr = {}
+    for f in numFields:
+        prop = _getMinMax(layer, f)
+        prop['properties']['count'] = count
+        attr[f.lower()] = prop
+        layer.ResetReading()
+    dataSource = None
+    info['attributes'] = attr
+    return info

dataqs/hifld/hifld.py

@@ -18,13 +18,15 @@
 ###############################################################################
 
 from __future__ import absolute_import
+import json
 import os
 import logging
 import time
 import traceback
 from django.conf import settings
 from dataqs.processor_base import GeoDataProcessor, DEFAULT_WORKSPACE
-from dataqs.helpers import ogr2ogr_exec, layer_exists, style_exists
+from dataqs.helpers import ogr2ogr_exec, layer_exists, style_exists,\
+    get_vector_layer_info
 from geonode.geoserver.helpers import ogc_server_settings
 
 logger = logging.getLogger("dataqs.processors")
@@ -90,6 +92,8 @@ def run(self):
                         lyr_file = os.path.join(
                             self.tmp_dir,
                             self.download(layer['url'], filename=table))
+                info = get_vector_layer_info(lyr_file)
+                layer_info = 'layer_info:{}'.format(json.dumps(info))
                 db = ogc_server_settings.datastore_db
                 ogr2ogr_exec("-overwrite -skipfailures -f PostgreSQL \
                     \"PG:host={db_host} user={db_user} password={db_pass} \
@@ -116,7 +120,7 @@ def run(self):
                     title=layer['name'],
                     description=layer['description'],
                     store=datastore,
-                    extra_keywords=[keywords])
+                    extra_keywords=[keywords, layer_info])
                 self.truncate_gs_cache(table)
             except Exception:
                 logger.error('Error with layer {}'.format(layer['name']))

dataqs/usgs_quakes/usgs_quakes.py

@@ -25,7 +25,7 @@
 from django.db import connections
 from dataqs.processor_base import GeoDataProcessor, DEFAULT_WORKSPACE
 from dataqs.helpers import postgres_query, ogr2ogr_exec, layer_exists, \
-    style_exists
+    style_exists, get_vector_layer_info
 from geonode.geoserver.helpers import ogc_server_settings
 
 logger = logging.getLogger("dataqs.processors")
@@ -106,7 +106,8 @@ def run(self, rss_file=None):
                     "%Y-%m-%d %H:%M:%S")
         with open(rss_file, 'w') as modified_file:
             json.dump(json_data, modified_file)
-
+        info = get_vector_layer_info(rss_file)
+        layer_info = 'layer_info:{}'.format(json.dumps(info))
         db = ogc_server_settings.datastore_db
         for table, title in zip(self.tables, self.titles):
             ogr2ogr_exec("-append -skipfailures -f PostgreSQL \
@@ -132,7 +133,8 @@ def run(self, rss_file=None):
                 title="Earthquakes - {}".format(title),
                 description=self.description,
                 store=datastore,
-                extra_keywords=['category:Geoscientific Information'])
+                extra_keywords=['category:Geoscientific Information',
+                                layer_info])
             self.truncate_gs_cache(table)
         self.purge_old_data()
         self.cleanup()