Merge pull request #29 from OpenGeoscience/gistemp

aashish24 · web-flow · commit bafeb2e72385 · 2016-09-07T23:10:10.000-04:00
Processor for NASA GISTEMP temperature anomaly data
diff --git a/ansible/roles/dataqs/tasks/main.yml b/ansible/roles/dataqs/tasks/main.yml
@@ -86,6 +86,12 @@
   command: "{{ app_code_dir }}/venvs/geonode/bin/python {{app_code_dir}}/venvs/geonode/src/dataqs/dataqs/gfms/gfms.py"
   ignore_errors: yes
 
+- include: geoserver_permissions.yml
+
+- name: Create the gistemp coverage store
+  command: "{{ app_code_dir }}/venvs/geonode/bin/python {{app_code_dir}}/venvs/geonode/src/dataqs/dataqs/gistemp/gistemp.py"
+  ignore_errors: yes
+
 - name: create geoserver data directory
   file: path=/var/lib/tomcat7/webapps/geoserver/data/data/geonode/forecast_io_airtemp recurse=yes owner=tomcat7 group=tomcat7 state=directory mode=g+rws
   sudo: yes
diff --git a/ansible/roles/dataqs/templates/dataq_settings.py b/ansible/roles/dataqs/templates/dataq_settings.py
@@ -34,6 +34,7 @@
     'dataqs.usgs_quakes',
     'dataqs.wqp',
     'dataqs.hifld',
+    'dataqs.gistemp',
 )
 
 # CELERY SETTINGS
@@ -121,6 +122,11 @@
         'task': 'dataqs.hifld.tasks.hifld_task',
         'schedule': crontab(day_of_week='sunday', hour=12, minute=0),
         'args': ()
+    },
+    'gistemp': {
+        'task': 'dataqs.gistemp.tasks.gistemp_task',
+        'schedule': crontab(day_of_month=15, hour=12, minute=0),
+        'args': ()
     }
 }
 
diff --git a/dataqs/gdacs/tests.py b/dataqs/gdacs/tests.py
@@ -24,6 +24,7 @@ def setUp(self):
 
     def tearDown(self):
         httpretty.disable()
+        self.processor.cleanup()
 
     def test_download(self):
         """
diff --git a/dataqs/gfms/tests.py b/dataqs/gfms/tests.py
@@ -12,7 +12,7 @@
 
 def get_mock_image():
     """
-    Return a canned response with HTML for Boston
+    Return a canned GFMS test image
     """
     zf = zipfile.ZipFile(os.path.join(script_dir,
                                       'resources/test_gfms.zip'))
@@ -33,6 +33,7 @@ def setUp(self):
 
     def tearDown(self):
         httpretty.disable()
+        self.processor.cleanup()
 
     def test_find_current(self):
         """
diff --git a/dataqs/gistemp/__init__.py b/dataqs/gistemp/__init__.py
diff --git a/dataqs/gistemp/gistemp.py b/dataqs/gistemp/gistemp.py
@@ -0,0 +1,107 @@
+from __future__ import absolute_import
+import logging
+import os
+import re
+import shutil
+from datetime import date
+from dateutil.relativedelta import relativedelta
+from dataqs.processor_base import GeoDataMosaicProcessor, GS_DATA_DIR
+from dataqs.helpers import get_band_count, gdal_translate, \
+    nc_convert, style_exists, cdo_fixlng, gunzip
+
+logger = logging.getLogger("dataqs.processors")
+script_dir = os.path.dirname(os.path.realpath(__file__))
+
+
+class GISTEMPProcessor(GeoDataMosaicProcessor):
+    """
+    Processor for Land-Ocean Temperature Index, ERSSTv4, 1200km smoothing
+    from the NASA Goddard Institute for Space Studies' Surface Temperature
+    Analysis (GISTEMP).
+    More info at http://data.giss.nasa.gov/gistemp/
+    """
+    prefix = "gistemp"
+    base_url = "http://data.giss.nasa.gov/pub/gistemp/gistemp1200_ERSSTv4.nc.gz"
+    layer_name = 'gistemp1200_ERSSTv4'
+    title = 'Global Monthly Air Temperature Anomalies, 1880/01/01 - {}'
+    abstract = """The GISTEMP analysis recalculates consistent temperature
+anomaly series from 1880 to the present for a regularly spaced array of virtual
+stations covering the whole globe. Those data are used to investigate regional
+and global patterns and trends.  Graphs and tables are updated around the
+middle of every month using current data files from NOAA GHCN v3 (meteorological
+ stations), ERSST v4 (ocean areas), and SCAR (Antarctic stations).
+
+ The displayed image is based on the most current month.
+
+Citations:
+    - GISTEMP Team, 2016: GISS Surface Temperature Analysis (GISTEMP).
+      NASA Goddard Institute for Space Studies.  Dataset accessed monthly
+      since 8/2016 at http://data.giss.nasa.gov/gistemp/.
+    - Hansen, J., R. Ruedy, M. Sato, and K. Lo, 2010: Global surface
+      temperature change, Rev. Geophys., 48, RG4004, doi:10.1029/2010RG000345.
+
+ """
+
+    def convert(self, nc_file):
+        nc_transform = nc_convert(nc_file)
+        cdo_transform = cdo_fixlng(nc_transform)
+        return cdo_transform
+
+    def extract_band(self, tif, band, outname):
+        outfile = os.path.join(self.tmp_dir, outname)
+        gdal_translate(tif, outfile, bands=[band],
+                       projection='EPSG:4326',
+                       options=['TILED=YES', 'COMPRESS=LZW'])
+        return outfile
+
+    def get_date(self, months):
+        start_month = date(1880, 1, 1)
+        return start_month + relativedelta(months=months-1)
+
+    def get_title(self, months):
+        end_month = self.get_date(months)
+        return self.title.format(end_month.strftime('%Y/%m/%d'))
+
+    def run(self):
+        """
+        Retrieve and process the latest NetCDF file.
+        """
+        gzfile = self.download(
+            self.base_url, '{}.nc.gz'.format(self.layer_name))
+        ncfile = gunzip(os.path.join(self.tmp_dir, gzfile))
+        cdf_file = self.convert(ncfile)
+        bands = get_band_count(cdf_file)
+        img_list = self.get_mosaic_filenames(self.layer_name)
+        for band in range(1, bands+1):
+            band_date = re.sub('[\-\.]+', '', self.get_date(band).isoformat())
+            img_name = '{}_{}T000000000Z.tif'.format(self.layer_name, band_date)
+            if img_name not in img_list:
+                band_tif = self.extract_band(cdf_file, band, img_name)
+                dst_file = self.data_dir.format(gsd=GS_DATA_DIR,
+                                                ws=self.workspace,
+                                                layer=self.layer_name,
+                                                file=img_name)
+                dst_dir = os.path.dirname(dst_file)
+                if not os.path.exists(dst_dir):
+                    os.makedirs(dst_dir)
+                if dst_file.endswith('.tif'):
+                    shutil.move(os.path.join(self.tmp_dir, band_tif), dst_file)
+                    self.post_geoserver(dst_file, self.layer_name)
+
+        if not style_exists(self.layer_name):
+            with open(os.path.join(script_dir,
+                                   'resources/gistemp.sld')) as sld:
+                self.set_default_style(self.layer_name, self.layer_name,
+                                       sld.read().format(latest_band=bands))
+        self.update_geonode(self.layer_name, title=self.get_title(bands),
+                            description=self.abstract,
+                            store=self.layer_name,
+                            bounds=('-180.0', '180.0', '-90.0', '90.0',
+                                    'EPSG:4326'))
+        self.truncate_gs_cache(self.layer_name)
+        self.cleanup()
+
+
+if __name__ == '__main__':
+    processor = GISTEMPProcessor()
+    processor.run()
diff --git a/dataqs/gistemp/resources/gistemp.sld b/dataqs/gistemp/resources/gistemp.sld
@@ -0,0 +1,34 @@
+<?xml version="1.0" encoding="UTF-8"?><sld:StyledLayerDescriptor xmlns="http://www.opengis.net/sld" xmlns:sld="http://www.opengis.net/sld" xmlns:ogc="http://www.opengis.net/ogc" xmlns:gml="http://www.opengis.net/gml" version="1.0.0">
+  <sld:NamedLayer>
+    <sld:Name>gistemp1200_ersstv4</sld:Name>
+    <sld:UserStyle>
+      <sld:Name>gistemp1200_ersstv4</sld:Name>
+      <sld:Title>gistemp1200_ersstv4</sld:Title>
+      <sld:FeatureTypeStyle>
+        <sld:Name>name</sld:Name>
+        <sld:Rule>
+          <sld:RasterSymbolizer>
+        <Opacity>1.0</Opacity>
+        <ChannelSelection>
+                <GrayChannel>
+                        <SourceChannelName>{latest_band}</SourceChannelName>
+                </GrayChannel>
+        </ChannelSelection>
+        <ColorMap extended="true">
+            <sld:ColorMapEntry color="#2b83ba" label="&lt;= -7.5 °C" opacity="1.0" quantity="-750"/>
+            <sld:ColorMapEntry color="#6bb0af" label="-5.0 °C" opacity="1.0" quantity="-500"/>
+            <sld:ColorMapEntry color="#abdda4" label="-3.75 °C" opacity="1.0" quantity="-375"/>
+            <sld:ColorMapEntry color="#d5eeb1" label="-1.5 °C" opacity="1.0" quantity="-150"/>
+            <sld:ColorMapEntry color="#ffffbf" label="0 °C" opacity="1.0" quantity="0"/>
+            <sld:ColorMapEntry color="#fed690" label="1.5 °C" opacity="1.0" quantity="150"/>
+            <sld:ColorMapEntry color="#fdae61" label="3.75 °C" opacity="1.0" quantity="375"/>
+            <sld:ColorMapEntry color="#ea633e" label="5.0 °C" opacity="1.0" quantity="500"/>
+            <sld:ColorMapEntry color="#d7191c" label="&gt;= 75 °C" opacity="1.0" quantity="750"/>
+            <sld:ColorMapEntry color="#FFFFFF" label="" opacity="0.0" quantity="32765"/>
+        </ColorMap>
+          </sld:RasterSymbolizer>
+        </sld:Rule>
+      </sld:FeatureTypeStyle>
+    </sld:UserStyle>
+  </sld:NamedLayer>
+</sld:StyledLayerDescriptor>
diff --git a/dataqs/gistemp/resources/gistemp1200_ERSSTv4.nc b/dataqs/gistemp/resources/gistemp1200_ERSSTv4.nc
diff --git a/dataqs/gistemp/tasks.py b/dataqs/gistemp/tasks.py
@@ -0,0 +1,10 @@
+from __future__ import absolute_import
+
+from celery import shared_task
+from dataqs.gistemp.gistemp import GISTEMPProcessor
+
+
+@shared_task
+def gistemp_task():
+    processor = GISTEMPProcessor()
+    processor.run()
diff --git a/dataqs/gistemp/tests.py b/dataqs/gistemp/tests.py
@@ -0,0 +1,57 @@
+import glob
+import os
+import httpretty
+from django.test import TestCase
+from dataqs.gistemp.gistemp import GISTEMPProcessor
+
+script_dir = os.path.dirname(os.path.realpath(__file__))
+
+
+def get_mock_image():
+    """
+    Return a canned test image (1 band of original NetCDF raster)
+    """
+    zf = os.path.join(script_dir, 'resources/gistemp1200_ERSSTv4.nc')
+    with open(zf, 'rb') as gzfile:
+        return gzfile.read()
+
+
+class GISTEMPTest(TestCase):
+    """
+    Tests the dataqs.gistemp module.  Since each processor is highly
+    dependent on a running GeoNode instance for most functions, only
+    independent functions are tested here.
+    """
+
+    def setUp(self):
+        self.processor = GISTEMPProcessor()
+        httpretty.enable()
+
+    def tearDown(self):
+        httpretty.disable()
+        self.processor.cleanup()
+
+    def test_download(self):
+        """
+        Verify that a file is downloaded
+        """
+        httpretty.register_uri(httpretty.GET,
+                               self.processor.base_url,
+                               body=get_mock_image())
+        imgfile = self.processor.download(
+            self.processor.base_url,
+            '{}.nc'.format(self.processor.layer_name))
+        self.assertTrue(os.path.exists(
+            os.path.join(self.processor.tmp_dir, imgfile)))
+
+    def test_cleanup(self):
+        httpretty.register_uri(httpretty.GET,
+                               self.processor.base_url,
+                               body=get_mock_image())
+        self.processor.download(self.processor.base_url,
+                                '{}.nc'.format(self.processor.layer_name))
+        self.assertNotEqual([], glob.glob(os.path.join(
+            self.processor.tmp_dir, self.processor.prefix + '*')))
+        self.processor.cleanup()
+        self.assertEquals([], glob.glob(os.path.join(
+            self.processor.tmp_dir, self.processor.prefix + '*')))
diff --git a/dataqs/helpers.py b/dataqs/helpers.py
@@ -1,5 +1,7 @@
 import datetime
+import gzip
 import logging
+import shutil
 import traceback
 import os
 import subprocess
@@ -105,6 +107,13 @@ def gdal_translate(src_filename, dst_filename, dst_format="GTiff", bands=None,
             os.remove(tmp_file)
 
 
+def gunzip(filepath):
+    outfile = filepath.rstrip('.gz')
+    with gzip.open(filepath) as gfile, open(outfile, 'wb') as ucfile:
+        shutil.copyfileobj(gfile, ucfile)
+    return outfile
+
+
 def nc_convert(filename):
     """
     Transform a NETCDF4 file to classic-model format.
@@ -113,9 +122,9 @@ def nc_convert(filename):
     :param filename:
     :return: output filename
     """
-    output_file = "{}.classic.nc".format(filename)
-    subprocess.check_call(["nccopy", "-k", "classic", "{}.nc".format(
-        filename), output_file])
+    output_file = re.sub('\.nc$', '.classic.nc', filename)
+    subprocess.check_call(
+        ["/usr/local/bin/nccopy", "-k", "classic", filename, output_file])
     return output_file
 
 
@@ -134,6 +143,19 @@ def cdo_invert(filename):
     return output_file
 
 
+def cdo_fixlng(filename):
+    """
+    Change the longitude coordinates of a NetCDF file from 0->360 to -180->180
+    :param filename: Full path * name of NetCDF image to process
+    :return: output filename
+    """
+
+    output_file = "{}.lng.nc".format(os.path.splitext(filename)[0])
+    subprocess.check_call(["cdo", "sellonlatbox,-180,180,-90,90", "{}".format(
+        filename), output_file])
+    return output_file
+
+
 def ogr2ogr_exec(argstring):
     """
     Run an ogr2ogr command
diff --git a/dataqs/nasa_gpm/nasa_gpm.py b/dataqs/nasa_gpm/nasa_gpm.py
@@ -1,5 +1,6 @@
 from __future__ import absolute_import
 
+import glob
 from ftplib import FTP
 import logging
 import os
@@ -100,6 +101,16 @@ def run(self, days=1):
         self.truncate_gs_cache(self.layer_name)
         self.cleanup()
 
+    def cleanup(self):
+        """
+        Remove any files in the temp directory matching
+        the processor class prefix or layer name
+        """
+        filelist = glob.glob("{}*.*".format(
+            os.path.join(self.tmp_dir, self.layer_name)))
+        for f in filelist:
+            os.remove(f)
+        super(GPMProcessor, self).cleanup()
 
 if __name__ == '__main__':
     processor = GPMProcessor()
diff --git a/dataqs/nasa_gpm/tests.py b/dataqs/nasa_gpm/tests.py
@@ -52,6 +52,9 @@ class NasaGpmTest(TestCase):
     def setUp(self):
         self.processor = GPMProcessor()
 
+    def tearDown(self):
+        self.processor.cleanup()
+
     @patch('ftplib.FTP', autospec=True)
     @patch('ftplib.FTP.retrbinary', mock_retrbinary)
     @patch('ftplib.FTP.nlst', mock_nlst)
diff --git a/dataqs/processor_base.py b/dataqs/processor_base.py
@@ -438,6 +438,28 @@ def drop_old_daily_images(self, nowtime, layer_name):
         mosaic_query = "ingestion<={}".format(month_cutoff)
         self.remove_mosaic_granules(mosaic_index_url, mosaic_query, layer_name)
 
+    def get_mosaic_filenames(self, layer_name):
+        """
+        Return a list of filenames in a mosaic
+        :param layer_name:
+        :return:
+        """
+        _user, _password = ogc_server_settings.credentials
+        mosaic_index_url = '{}.json'.format(
+            self.mosaic_url.format(ogc_server_settings.hostname,
+                                   self.workspace,
+                                   layer_name,
+                                   layer_name)
+        )
+        files = []
+        try:
+            r = requests.get(
+                mosaic_index_url, timeout=30, auth=(_user, _password))
+            for feature in r.json()['features']:
+                files.append(feature['properties']['location'])
+        finally:
+            return files
+
     def create_mosaic_properties_zip(self, layer_name, img_file):
         """
         Create a zipfile containing the required config files and
diff --git a/dataqs/wqp/tests.py b/dataqs/wqp/tests.py
@@ -32,6 +32,7 @@ def setUp(self):
 
     def tearDown(self):
         httpretty.disable()
+        self.processor.cleanup()
 
     def test_download(self):
         """
diff --git a/local_settings.py b/local_settings.py

Original file line number	Diff line number	Diff line change
`@@ -34,6 +34,7 @@`
`34`	`34`	`'dataqs.usgs_quakes',`
`35`	`35`	`'dataqs.wqp',`
`36`	`36`	`'dataqs.hifld',`
	`37`	`+ 'dataqs.gistemp',`
`37`	`38`	`)`
`38`	`39`
`39`	`40`	`# CELERY SETTINGS`
`@@ -121,6 +122,11 @@`
`121`	`122`	`'task': 'dataqs.hifld.tasks.hifld_task',`
`122`	`123`	`'schedule': crontab(day_of_week='sunday', hour=12, minute=0),`
`123`	`124`	`'args': ()`
	`125`	`+ },`
	`126`	`+ 'gistemp': {`
	`127`	`+ 'task': 'dataqs.gistemp.tasks.gistemp_task',`
	`128`	`+ 'schedule': crontab(day_of_month=15, hour=12, minute=0),`
	`129`	`+ 'args': ()`
`124`	`130`	`}`
`125`	`131`	`}`
`126`	`132`