|
| 1 | +#!/usr/bin/env python |
| 2 | +# -*- coding: utf-8 -*- |
| 3 | + |
| 4 | +############################################################################### |
| 5 | +# Copyright Kitware Inc. and Epidemico Inc. |
| 6 | +# |
| 7 | +# Licensed under the Apache License, Version 2.0 ( the "License" ); |
| 8 | +# you may not use this file except in compliance with the License. |
| 9 | +# You may obtain a copy of the License at |
| 10 | +# |
| 11 | +# http://www.apache.org/licenses/LICENSE-2.0 |
| 12 | +# |
| 13 | +# Unless required by applicable law or agreed to in writing, software |
| 14 | +# distributed under the License is distributed on an "AS IS" BASIS, |
| 15 | +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 16 | +# See the License for the specific language governing permissions and |
| 17 | +# limitations under the License. |
| 18 | +############################################################################### |
| 19 | + |
| 20 | +from __future__ import absolute_import |
| 21 | +import logging |
| 22 | +import os |
| 23 | +import re |
| 24 | +import shutil |
| 25 | +from datetime import date |
| 26 | +from ftplib import FTP |
| 27 | + |
| 28 | +from dateutil.relativedelta import relativedelta |
| 29 | +from dataqs.processor_base import GeoDataMosaicProcessor, GS_DATA_DIR, \ |
| 30 | + GS_TMP_DIR |
| 31 | +from dataqs.helpers import get_band_count, gdal_translate, \ |
| 32 | + nc_convert, style_exists, cdo_fixlng |
| 33 | + |
| 34 | +logger = logging.getLogger("dataqs.processors") |
| 35 | +script_dir = os.path.dirname(os.path.realpath(__file__)) |
| 36 | + |
| 37 | + |
| 38 | +class UoDAirTempPrecipProcessor(GeoDataMosaicProcessor): |
| 39 | + """ |
| 40 | + Processor for Land-Ocean Temperature Index, ERSSTv4, 1200km smoothing |
| 41 | + from the NASA Goddard Institute for Space Studies' Surface Temperature |
| 42 | + Analysis (GISTEMP). |
| 43 | + More info at http://data.giss.nasa.gov/gistemp/ |
| 44 | + """ |
| 45 | + prefix = "uod_" |
| 46 | + base_url = "ftp.cdc.noaa.gov" |
| 47 | + |
| 48 | + layers = { |
| 49 | + 'air.mon.mean.v401.nc': { |
| 50 | + 'title': 'U. Delaware Monthly Mean Air Temperature 1901 - 2015', |
| 51 | + 'name': 'uod_air_mean_401' |
| 52 | + }, |
| 53 | + 'precip.mon.total.v401.nc': { |
| 54 | + 'title': 'U. Delaware Monthly Total Precipitation 1901 - 2015', |
| 55 | + 'name': 'uod_precip_total_401' |
| 56 | + } |
| 57 | + |
| 58 | + } |
| 59 | + abstract = """Cort Willmott & Kenji Matsuura of the University of Delaware |
| 60 | + have put data together from a large number of stations, both from the GHCN2 |
| 61 | + (Global Historical Climate Network) and, more extensively, from the archive |
| 62 | + of Legates & Willmott. More details can be found here for temperature and |
| 63 | + here for precipitation. The result is a monthly climatology of precipitation |
| 64 | + and air temperature, both at the surface, and a time series, spanning 1900 |
| 65 | + to 2010, of monthly mean surface air temperatures, and monthly total |
| 66 | + precipitation. It is land-only in coverage, and complements the ICOADS |
| 67 | + (International Comprehensive Ocean-Atmosphere Data Set) data set well. For a |
| 68 | + complete description of the data as given by the providers, related |
| 69 | + datasets and references to relevant papers please see their web pages at the |
| 70 | + University of Delaware.. |
| 71 | +
|
| 72 | + Source: http://www.esrl.noaa.gov/psd/data/gridded/data.UDel_AirT_Precip.html |
| 73 | +
|
| 74 | + The displayed image is based on the most current month. |
| 75 | +
|
| 76 | +Citations: |
| 77 | + - Willmott, C. J. and K. Matsuura (2001) Terrestrial Air Temperature and |
| 78 | + Precipitation: Monthly and Annual Time Series (1950 - 1999), |
| 79 | + http://climate.geog.udel.edu/~climate/html_pages/README.ghcn_ts2.html. |
| 80 | + - UDel_AirT_Precip data provided by the NOAA/OAR/ESRL PSD, Boulder, |
| 81 | + Colorado, USA, from their Web site at http://www.esrl.noaa.gov/psd/ |
| 82 | + """ |
| 83 | + |
| 84 | + def download(self, tmp_dir=GS_TMP_DIR): |
| 85 | + """ |
| 86 | + Retrieve NetCDF files via FTP |
| 87 | + :param tmp_dir: Temp directory to store files |
| 88 | + :return: list of saved output files |
| 89 | + """ |
| 90 | + ftp = FTP(self.base_url) |
| 91 | + ftp.login('anonymous', 'anonymous') |
| 92 | + ftp.cwd('/Datasets/udel.airt.precip/') |
| 93 | + outfiles = [] |
| 94 | + for file in self.layers.keys(): |
| 95 | + outfile = os.path.join(tmp_dir, '{}{}'.format(self.prefix, file)) |
| 96 | + with open(outfile, 'wb') as output: |
| 97 | + ftp.retrbinary('RETR %s' % file, output.write) |
| 98 | + outfiles.append(outfile) |
| 99 | + return outfiles |
| 100 | + |
| 101 | + def convert(self, nc_file): |
| 102 | + nc_transform = nc_convert(nc_file) |
| 103 | + cdo_transform = cdo_fixlng(nc_transform) |
| 104 | + return cdo_transform |
| 105 | + |
| 106 | + def extract_band(self, tif, band, outname): |
| 107 | + outfile = os.path.join(self.tmp_dir, outname) |
| 108 | + gdal_translate(tif, outfile, bands=[band], |
| 109 | + projection='EPSG:4326', |
| 110 | + options=['TILED=YES', 'COMPRESS=LZW']) |
| 111 | + return outfile |
| 112 | + |
| 113 | + def get_date(self, months): |
| 114 | + start_month = date(1901, 1, 1) |
| 115 | + return start_month + relativedelta(months=months - 1) |
| 116 | + |
| 117 | + def run(self): |
| 118 | + """ |
| 119 | + Retrieve and process the latest NetCDF file. |
| 120 | + """ |
| 121 | + cdf_files = self.download() |
| 122 | + for cdf in cdf_files: |
| 123 | + cdf_file = self.convert(cdf) |
| 124 | + bands = get_band_count(cdf_file) |
| 125 | + key = os.path.basename(cdf).lstrip(self.prefix) |
| 126 | + print(key) |
| 127 | + layer_name = self.layers[key]['name'] |
| 128 | + img_list = self.get_mosaic_filenames(layer_name) |
| 129 | + for band in range(1, bands + 1): |
| 130 | + band_date = re.sub('[\-\.]+', '', |
| 131 | + self.get_date(band).isoformat()) |
| 132 | + img_name = '{}_{}T000000000Z.tif'.format(layer_name, band_date) |
| 133 | + if img_name not in img_list: |
| 134 | + band_tif = self.extract_band(cdf_file, band, img_name) |
| 135 | + dst_file = self.data_dir.format(gsd=GS_DATA_DIR, |
| 136 | + ws=self.workspace, |
| 137 | + layer=layer_name, |
| 138 | + file=img_name) |
| 139 | + dst_dir = os.path.dirname(dst_file) |
| 140 | + if not os.path.exists(dst_dir): |
| 141 | + os.makedirs(dst_dir) |
| 142 | + if dst_file.endswith('.tif'): |
| 143 | + shutil.move(os.path.join(self.tmp_dir, band_tif), |
| 144 | + dst_file) |
| 145 | + self.post_geoserver(dst_file, layer_name) |
| 146 | + |
| 147 | + if not style_exists(layer_name): |
| 148 | + with open(os.path.join(script_dir, 'resources/{}.sld'.format( |
| 149 | + layer_name))) as sld: |
| 150 | + print(layer_name, |
| 151 | + os.path.join(script_dir, |
| 152 | + 'resources/{}.sld'.format(layer_name))) |
| 153 | + self.set_default_style(layer_name, layer_name, sld.read()) |
| 154 | + self.update_geonode(layer_name, |
| 155 | + title=self.layers[key]['title'], |
| 156 | + description=self.abstract, |
| 157 | + store=layer_name, |
| 158 | + bounds=('-180.0', '180.0', '-90.0', '90.0', |
| 159 | + 'EPSG:4326')) |
| 160 | + self.truncate_gs_cache(layer_name) |
| 161 | + self.cleanup() |
| 162 | + |
| 163 | + |
| 164 | +if __name__ == '__main__': |
| 165 | + processor = UoDAirTempPrecipProcessor() |
| 166 | + processor.run() |
0 commit comments