Merge pull request #41 from dorukozturk/master

dorukozturk · web-flow · commit 78eff167a071 · 2017-01-10T14:42:14.000-05:00
add-keywords-for-categorizing
diff --git a/ansible/roles/dataqs/tasks/main.yml b/ansible/roles/dataqs/tasks/main.yml
@@ -7,6 +7,7 @@
     - libgdal-dev
     - cdo
     - netcdf-bin
+    - libffi-dev
   sudo: yes
 
 - name: install python dependencies
@@ -15,6 +16,11 @@
     - redis
     - flower
 
+- name: upgrade urllib3
+  pip: virtualenv={{virtualenv_dir}}/{{app_name}} name={{item}} state=forcereinstall
+  with_items:
+    - urllib3[secure]
+
 - name: install dataqs python package
   pip: virtualenv={{virtualenv_dir}}/{{app_name}} name=git+https://github.com/OpenGeoscience/dataqs.git#egg=dataqs
   environment:
@@ -67,30 +73,35 @@
 - name: Create the usgs_quakes  store
   command: "{{ app_code_dir }}/venvs/geonode/bin/python {{app_code_dir}}/venvs/geonode/src/dataqs/dataqs/usgs_quakes/usgs_quakes.py"
   ignore_errors: yes
+  sudo: yes
 
 - include: geoserver_permissions.yml
 
 - name: Create the wqp store
   command: "{{ app_code_dir }}/venvs/geonode/bin/python {{app_code_dir}}/venvs/geonode/src/dataqs/dataqs/wqp/wqp.py"
   ignore_errors: yes
+  sudo: yes
 
 - include: geoserver_permissions.yml
 
 - name: Create the gdacs store
   command: "{{ app_code_dir }}/venvs/geonode/bin/python {{app_code_dir}}/venvs/geonode/src/dataqs/dataqs/gdacs/gdacs.py"
   ignore_errors: yes
+  sudo: yes
 
 - include: geoserver_permissions.yml
 
 - name: Create the gfms coverage store
   command: "{{ app_code_dir }}/venvs/geonode/bin/python {{app_code_dir}}/venvs/geonode/src/dataqs/dataqs/gfms/gfms.py"
   ignore_errors: yes
+  sudo: yes
 
 - include: geoserver_permissions.yml
 
 - name: Create the gistemp coverage store
   command: "{{ app_code_dir }}/venvs/geonode/bin/python {{app_code_dir}}/venvs/geonode/src/dataqs/dataqs/gistemp/gistemp.py"
   ignore_errors: yes
+  sudo: yes
 
 - name: create geoserver data directory
   file: path=/var/lib/tomcat7/webapps/geoserver/data/data/geonode/forecast_io_airtemp recurse=yes owner=tomcat7 group=tomcat7 state=directory mode=774
@@ -101,24 +112,28 @@
 - name: Create the forecastio coverage store
   command: "{{ app_code_dir }}/venvs/geonode/bin/python {{app_code_dir}}/venvs/geonode/src/dataqs/dataqs/forecastio/forecastio_air.py"
   ignore_errors: yes
+  sudo: yes
 
 - include: geoserver_permissions.yml
 
 - name: Create the hifld layers
   command: "{{ app_code_dir }}/venvs/geonode/bin/python {{app_code_dir}}/venvs/geonode/src/dataqs/dataqs/hifld/hifld.py"
   ignore_errors: yes
+  sudo: yes
 
 - include: geoserver_permissions.yml
 
 - name: Create the cmap layer
   command: "{{ app_code_dir }}/venvs/geonode/bin/python {{app_code_dir}}/venvs/geonode/src/dataqs/dataqs/cmap/cmap.py"
   ignore_errors: yes
+  sudo: yes
 
 - include: geoserver_permissions.yml
 
 - name: Create the landscan layer
   command: "{{ app_code_dir }}/venvs/geonode/bin/python {{app_code_dir}}/venvs/geonode/src/dataqs/dataqs/landscan/landscan.py"
   ignore_errors: yes
+  sudo: yes
 
 - name: Django updatelayers
   django_manage: command=updatelayers
diff --git a/dataqs/airnow/airnow.py b/dataqs/airnow/airnow.py
@@ -161,8 +161,10 @@ def run(self, days=1):
                 with open(os.path.join(
                         script_dir, 'resources/airnow.sld')) as sld:
                     self.set_default_style(layer_name, layer_name, sld.read())
-            self.update_geonode(layer_name, title=layer_title,
-                                description=self.description, store=layer_name)
+            self.update_geonode(
+                layer_name, title=layer_title,
+                description=self.description, store=layer_name,
+                extra_keywords=['category:Climatology Meteorology Atmosphere'])
             self.truncate_gs_cache(layer_name)
         self.cleanup()
 
diff --git a/dataqs/aqicn/aqicn.py b/dataqs/aqicn/aqicn.py
@@ -326,10 +326,12 @@ def run(self):
         if not style_exists(layer_name):
             with open(os.path.join(script_dir, 'resources/aqicn.sld')) as sld:
                 self.set_default_style(layer_name, layer_name, sld.read())
-        self.update_geonode(layer_name,
-                            title='Air Quality Index',
-                            description=self.description,
-                            store=datastore)
+        self.update_geonode(
+            layer_name,
+            title='Air Quality Index',
+            description=self.description,
+            store=datastore,
+            extra_keywords=['category:Climatology Meteorology Atmosphere'])
         self.truncate_gs_cache(layer_name)
         self.cleanup()
 
diff --git a/dataqs/cmap/cmap.py b/dataqs/cmap/cmap.py
@@ -143,11 +143,13 @@ def run(self):
                                    'resources/cmap.sld')) as sld:
                 self.set_default_style(self.layer_name, self.layer_name,
                                        sld.read().format(latest_band=bands))
-        self.update_geonode(self.layer_name, title=self.get_title(bands),
-                            description=self.abstract,
-                            store=self.layer_name,
-                            bounds=('-178.75', '178.75', '-88.75', '88.75',
-                                    'EPSG:4326'))
+        self.update_geonode(
+            self.layer_name, title=self.get_title(bands),
+            description=self.abstract,
+            store=self.layer_name,
+            bounds=('-178.75', '178.75', '-88.75', '88.75',
+                    'EPSG:4326'),
+            extra_keywords=['category:Climatology Meteorology Atmosphere'])
         self.truncate_gs_cache(self.layer_name)
         self.cleanup()
 
diff --git a/dataqs/forecastio/forecastio_air.py b/dataqs/forecastio/forecastio_air.py
@@ -119,11 +119,13 @@ def run(self, now=None):
         self.drop_old_hourly_images(now, self.layer_name)
         self.drop_old_daily_images(now, self.layer_name)
 
-        self.update_geonode(self.layer_name, title=self.parse_name(now),
-                            description=self.description,
-                            store=self.layer_name,
-                            bounds=('-180.0', '180.0',
-                                    '-90.0', '90.0', 'EPSG:4326'))
+        self.update_geonode(
+            self.layer_name, title=self.parse_name(now),
+            description=self.description,
+            store=self.layer_name,
+            bounds=('-180.0', '180.0',
+                    '-90.0', '90.0', 'EPSG:4326'),
+            extra_keywords=['category:Climatology Meteorology Atmosphere'])
         self.truncate_gs_cache(self.layer_name)
         self.cleanup()
 
diff --git a/dataqs/gdacs/gdacs.py b/dataqs/gdacs/gdacs.py
@@ -98,7 +98,8 @@ def run(self):
                     script_dir, 'resources/gdacs.sld')) as sld:
                 self.set_default_style(self.prefix, self.prefix, sld.read())
         self.update_geonode(self.prefix, title=self.layer_title,
-                            description=self.description, store=datastore)
+                            description=self.description, store=datastore,
+                            extra_keywords=['category:Disaster Alerts'])
         self.truncate_gs_cache(self.prefix)
         self.cleanup()
 
diff --git a/dataqs/gfms/gfms.py b/dataqs/gfms/gfms.py
@@ -166,7 +166,9 @@ def import_future(self):
                 self.set_default_style(self.layer_future,
                                        self.layer_future, sld.read())
         self.update_geonode(self.layer_future, title=new_title,
-                            store=self.layer_future)
+                            store=self.layer_future,
+                            description=self.description,
+                            extra_keywords=['category:Disaster Alerts'])
         self.truncate_gs_cache(self.layer_future)
 
     def import_current(self):
@@ -185,7 +187,8 @@ def import_current(self):
                                        self.layer_current, sld.read())
         self.update_geonode(self.layer_current, title=new_title,
                             store=self.layer_current,
-                            description=self.description)
+                            description=self.description,
+                            extra_keywords=['category:Disaster Alerts'])
         self.truncate_gs_cache(self.layer_current)
 
     def run(self):
diff --git a/dataqs/gistemp/gistemp.py b/dataqs/gistemp/gistemp.py
@@ -112,11 +112,13 @@ def run(self):
                                    'resources/gistemp.sld')) as sld:
                 self.set_default_style(self.layer_name, self.layer_name,
                                        sld.read().format(latest_band=bands))
-        self.update_geonode(self.layer_name, title=self.get_title(bands),
-                            description=self.abstract,
-                            store=self.layer_name,
-                            bounds=('-180.0', '180.0', '-90.0', '90.0',
-                                    'EPSG:4326'))
+        self.update_geonode(
+            self.layer_name, title=self.get_title(bands),
+            description=self.abstract,
+            store=self.layer_name,
+            bounds=('-180.0', '180.0', '-90.0', '90.0',
+                    'EPSG:4326'),
+            extra_keywords=['category:Climatology Meteorology Atmosphere'])
         self.truncate_gs_cache(self.layer_name)
         self.cleanup()
 
diff --git a/dataqs/hifld/hifld.py b/dataqs/hifld/hifld.py
@@ -39,6 +39,19 @@ class HIFLDProcessor(GeoDataProcessor):
     prefix = 'hifld_'
     layers = []
     base_url = "https://hifld-dhs-gii.opendata.arcgis.com/datasets/"
+    layer_category_mapping = {
+        'us_state_boundaries': 'category:Boundaries',
+        'us_county_boundaries': 'category:Boundaries',
+        'us_urban_areas': 'category:Boundaries',
+        'poultry_facilities': 'category:Agriculture',
+        'state_fairgrounds': 'category:Agriculture',
+        'epa_tsca_facilities': 'category:Chemicals',
+        'epa_er_rmp_facilities': 'category:Chemicals',
+        'epa_er_tri_facilities': 'category:Chemicals',
+        'hospitals': 'category:Public Health',
+        'pharmacies': 'category:Public Health',
+        'hazmat_routes': 'category:Chemicals:'
+    }
 
     def __init__(self, layers=None):
         super(HIFLDProcessor, self).__init__()
@@ -87,10 +100,13 @@ def run(self):
                         sld_text = sld.read().format(table=layer['table'],
                                                      title=layer['name'])
                         self.set_default_style(table, table, sld_text)
-                self.update_geonode(table,
-                                    title=layer['name'],
-                                    description=layer['description'],
-                                    store=datastore)
+                keywords = self.layer_category_mapping[layer['table']]
+                self.update_geonode(
+                    table,
+                    title=layer['name'],
+                    description=layer['description'],
+                    store=datastore,
+                    extra_keywords=[keywords])
                 self.truncate_gs_cache(table)
             except Exception:
                 logger.error('Error with layer {}'.format(layer['name']))
diff --git a/dataqs/landscan/landscan.py b/dataqs/landscan/landscan.py
@@ -94,7 +94,8 @@ def import_landscan(self, landscan_tiff):
                                        self.layer, sld.read())
         self.update_geonode(self.layer, title=self.layer,
                             store=self.layer,
-                            description=self.description)
+                            description=self.description,
+                            extra_keywords=['category:Population'])
         self.truncate_gs_cache(self.layer)
 
     def run(self):
diff --git a/dataqs/mmwr/mmwr.py b/dataqs/mmwr/mmwr.py
@@ -196,7 +196,8 @@ def update_layer(self, layer):
         self.update_geonode(
             table,
             title='{} {}'.format(self.base_title, layer),
-            description=self.description)
+            description=self.description,
+            extra_keywords=['category:Population'])
         self.truncate_gs_cache(table)
 
     def run(self):
diff --git a/dataqs/nasa_gpm/nasa_gpm.py b/dataqs/nasa_gpm/nasa_gpm.py
@@ -124,12 +124,14 @@ def run(self, days=1):
             with open(os.path.join(script_dir, 'resources/gpm.sld')) as sld:
                 self.set_default_style(self.layer_name,
                                        self.layer_name, sld.read())
-        self.update_geonode(self.layer_name,
-                            title=layer_title,
-                            description=self.description,
-                            store=self.layer_name,
-                            bounds=('-180.0', '180.0', '-90.0', '90.0',
-                                    'EPSG:4326'))
+        self.update_geonode(
+            self.layer_name,
+            title=layer_title,
+            description=self.description,
+            store=self.layer_name,
+            bounds=('-180.0', '180.0', '-90.0', '90.0',
+                    'EPSG:4326'),
+            extra_keywords=['category:Climatology Meteorology Atmosphere'])
         self.truncate_gs_cache(self.layer_name)
         self.cleanup()
 
diff --git a/dataqs/processor_base.py b/dataqs/processor_base.py
@@ -235,7 +235,8 @@ def update_gs_metadata(self, layer_name, json_data, vector=False,
         return res.content
 
     def update_geonode(self, layer_name, title="", description="",
-                       category=None, bounds=None, store=None):
+                       category=None, bounds=None, store=None,
+                       extra_keywords=None):
         """
         Update a layer and it's title in GeoNode
         :param layer_name: Name of the layer
@@ -265,8 +266,19 @@ def update_geonode(self, layer_name, title="", description="",
                 url = ogc_server_settings.rest
                 gs_catalog = Catalog(url, _user, _password)
                 gs_catalog.save(res)
+            if extra_keywords:
+                assert isinstance(extra_keywords, list)
+                # Append extra keywords to the default ones
+                res = lyr.gs_resource
+                keywords = res.keywords + extra_keywords
+                res.keywords = keywords
+                _user, _password = ogc_server_settings.credentials
+                url = ogc_server_settings.rest
+                gs_catalog = Catalog(url, _user, _password)
+                gs_catalog.save(res)
 
-    def set_default_style(self, layer_name, sld_name, sld_content, create=True):
+    def set_default_style(self, layer_name, sld_name, sld_content,
+                          create=True):
         """
         Create a style and assign it as default to a layer
         :param layer_name: the layer to assign the style to
diff --git a/dataqs/spei/spei.py b/dataqs/spei/spei.py
@@ -85,7 +85,8 @@ def run(self):
             self.update_geonode(layer_name,
                                 title=self.spei_files[layer_name],
                                 description=self.description,
-                                store=layer_name)
+                                store=layer_name,
+                                extra_keywords=['category:Agriculture'])
             self.truncate_gs_cache(layer_name)
             self.cleanup()
 
diff --git a/dataqs/usgs_quakes/usgs_quakes.py b/dataqs/usgs_quakes/usgs_quakes.py
@@ -127,10 +127,12 @@ def run(self, rss_file=None):
                 with open(os.path.join(
                         script_dir, 'resources/usgs.sld')) as sld:
                     self.set_default_style(table, table, sld.read())
-            self.update_geonode(table,
-                                title="Earthquakes - {}".format(title),
-                                description=self.description,
-                                store=datastore)
+            self.update_geonode(
+                table,
+                title="Earthquakes - {}".format(title),
+                description=self.description,
+                store=datastore,
+                extra_keywords=['category:Geoscientific Information'])
             self.truncate_gs_cache(table)
         self.purge_old_data()
         self.cleanup()
diff --git a/dataqs/whisp/whisp.py b/dataqs/whisp/whisp.py
@@ -172,7 +172,8 @@ def run(self):
                     self.set_default_style(self.prefix, self.prefix, sld.read())
         self.update_geonode(self.prefix,
                             title=self.title,
-                            description=self.description)
+                            description=self.description,
+                            extra_keywords=['category:Wildlife'])
         self.truncate_gs_cache(self.prefix)
         self.cleanup()
 
diff --git a/dataqs/worldclim/worldclim.py b/dataqs/worldclim/worldclim.py
@@ -144,8 +144,10 @@ def publish(self, tif, name, title, desc):
             self.set_default_style(name, style, sld.read(),
                                    create=not style_exists(style))
         self.truncate_gs_cache(name)
-        self.update_geonode(name, title,
-                            description=desc, category=category, store=name)
+        self.update_geonode(
+            name, title,
+            description=desc, category=category, store=name,
+            extra_keywords=['category:Climatology Meteorology Atmosphere'])
 
     def cleanup(self, outdir):
         """
diff --git a/dataqs/wqp/wqp.py b/dataqs/wqp/wqp.py
@@ -263,7 +263,8 @@ def run(self):
                 self.update_geonode(layer_name,
                                     title=layer_title,
                                     description=self.description,
-                                    store=datastore)
+                                    store=datastore,
+                                    extra_keywords=['category:Water Quality'])
                 self.truncate_gs_cache(layer_name)
         self.cleanup()
 
