Descriptions for processors; headers for files.

Author: Matt Bertrand

dataqs/airnow/airnow.py

@@ -1,5 +1,23 @@
-from __future__ import absolute_import
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+###############################################################################
+#  Copyright Kitware Inc. and Epidemico Inc.
+#
+#  Licensed under the Apache License, Version 2.0 ( the "License" );
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+###############################################################################
 
+from __future__ import absolute_import
 from ftplib import FTP
 import logging
 import os
@@ -32,6 +50,31 @@ class AirNowGRIB2HourlyProcessor(GeoDataMosaicProcessor):
                    "airnow_aqi_combined"]
     img_patterns = ["", "_pm25", "_combined"]
     layer_titles = ["Ozone", "PM25", "Combined Ozone & PM25"]
+    description = """
+U.S. Environmental Protection Agency’s (EPA) nationwide, voluntary program,
+AirNow(www.airnow.gov), provides real-time air quality data and forecasts to
+protect public health across the United States, Canada, and parts of Mexico.
+AirNowreceives real-time ozone and PM2.5data from over 2,000 monitors and
+collects air quality forecasts for more than 300 cities.
+
+As part of the Global Earth Observation System of Systems (GEOSS)
+(www.epa.gov/geoss) program, the AirNow API system broadens access to AirNowdata
+ and data products. AirNow API produces data products in several standard data
+formats and makes them available via FTP and web services. This document
+describes the GRIB2 file formats.
+
+All data provided by AirNow API are made possible by the efforts of more than
+120 local, state, tribal, provincial, and federal government agencies
+(www.airnow.gov/index.cfm?action=airnow.partnerslist). These data are not fully
+verified or validated and should be considered preliminary and subject to
+change. Data and information reported to AirNow from federal, state, local, and
+tribal agencies are for the express purpose of reporting and forecasting the
+Air Quality Index (AQI). As such, they should not be used to formulate or
+support regulation, trends, guidance, or any other government or public
+decision making. Official regulatory air quality data must be obtained from
+EPA’s Air Quality System (AQS) (www.epa.gov/ttn/airs/airsaqs). See the AirNow
+Data Exchange Guidelines at http://airnowapi.org/docs/DataUseGuidelines.pdf.
+"""
 
     def download(self, auth_account=AIRNOW_ACCOUNT, days=1):
         """
@@ -118,7 +161,8 @@ def run(self, days=1):
                 with open(os.path.join(
                         script_dir, 'resources/airnow.sld')) as sld:
                     self.set_default_style(layer_name, layer_name, sld.read())
-            self.update_geonode(layer_name, title=layer_title, store=layer_name)
+            self.update_geonode(layer_name, title=layer_title,
+                                description=self.description, store=layer_name)
             self.truncate_gs_cache(layer_name)
         self.cleanup()
 

dataqs/airnow/tasks.py

@@ -1,3 +1,22 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+###############################################################################
+#  Copyright Kitware Inc. and Epidemico Inc.
+#
+#  Licensed under the Apache License, Version 2.0 ( the "License" );
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+###############################################################################
+
 from __future__ import absolute_import
 
 from celery import shared_task

dataqs/airnow/tests.py

@@ -1,3 +1,22 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+###############################################################################
+#  Copyright Kitware Inc. and Epidemico Inc.
+#
+#  Licensed under the Apache License, Version 2.0 ( the "License" );
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+###############################################################################
+
 import glob
 import zipfile
 import os

dataqs/aqicn/aqicn.py

@@ -1,3 +1,22 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+###############################################################################
+#  Copyright Kitware Inc. and Epidemico Inc.
+#
+#  Licensed under the Apache License, Version 2.0 ( the "License" );
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+###############################################################################
+
 import json
 import logging
 import os
@@ -100,7 +119,7 @@ def __init__(self, table, cities):
     def handle_city(self, i, city):
         try:
             logger.debug('Scraping %d of %d cities - %s' % (
-                i+1, len(self.cities), city['url']))
+                i + 1, len(self.cities), city['url']))
             time.sleep(randint(1, self.max_wait))
             page = requests.get(city['url'], timeout=60, headers=REQ_HEADER)
             page.raise_for_status()
@@ -241,6 +260,8 @@ class AQICNProcessor(GeoDataProcessor):
         'uvi': 'Ultraviolet Index',
         'w': 'Wind Speed'
     }
+    description = \
+        'Air quality index data scraped from http://aqicn.org every 3 hours.'
 
     def __init__(self, countries=None, cities=None):
         super(AQICNProcessor, self).__init__()
@@ -279,9 +300,9 @@ def split_list(self, num):
         """ Yield n successive lists from cities list.
         """
         newn = int(1.0 * len(self.cities) / num + 0.5)
-        for i in xrange(0, num-1):
-            yield self.cities[i*newn:i*newn+newn]
-        yield self.cities[num*newn-newn:]
+        for i in xrange(0, num - 1):
+            yield self.cities[i * newn:i * newn + newn]
+        yield self.cities[num * newn - newn:]
 
     def process(self):
         if not table_exists(self.prefix):
@@ -307,10 +328,12 @@ def run(self):
                 self.set_default_style(layer_name, layer_name, sld.read())
         self.update_geonode(layer_name,
                             title='Air Quality Index',
+                            description=self.description,
                             store=datastore)
         self.truncate_gs_cache(layer_name)
         self.cleanup()
 
+
 if __name__ == '__main__':
     parser = AQICNProcessor()
     parser.run()

dataqs/aqicn/tasks.py

@@ -1,3 +1,22 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+###############################################################################
+#  Copyright Kitware Inc. and Epidemico Inc.
+#
+#  Licensed under the Apache License, Version 2.0 ( the "License" );
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+###############################################################################
+
 from __future__ import absolute_import
 from celery import shared_task
 from dataqs.aqicn.aqicn import AQICNProcessor

dataqs/aqicn/tests.py

@@ -1,3 +1,22 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+###############################################################################
+#  Copyright Kitware Inc. and Epidemico Inc.
+#
+#  Licensed under the Apache License, Version 2.0 ( the "License" );
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+###############################################################################
+
 import json
 import os
 import datetime

dataqs/csv_helpers.py

@@ -1,3 +1,22 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+###############################################################################
+#  Copyright Kitware Inc. and Epidemico Inc.
+#
+#  Licensed under the Apache License, Version 2.0 ( the "License" );
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+###############################################################################
+
 import csv
 import codecs
 import cStringIO

dataqs/forecastio/forecastio_air.py

@@ -1,3 +1,22 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+###############################################################################
+#  Copyright Kitware Inc. and Epidemico Inc.
+#
+#  Licensed under the Apache License, Version 2.0 ( the "License" );
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+###############################################################################
+
 from __future__ import absolute_import
 
 import logging
@@ -22,6 +41,13 @@ class ForecastIOAirTempProcessor(GeoDataMosaicProcessor):
     prefix = "forecast_io_airtemp"
     base_url = "http://maps.forecast.io/temperature/"
     layer_name = "forecast_io_airtemp"
+    description = """Project Quicksilver is an experimental new data product
+that attempts to create the world's highest resolution real-time map of global
+(near-surface) air temperature.\n\n
+It is generated using the same source data models that power Forecast.io,
+combined with a sophisticated microclimate model that adjusts the temperatures
+based on the effects of elevation, terrain, proximity to water, foliage cover,
+and other factors.\n\nSource: http://blog.forecast.io/project-quicksilver/"""
 
     def parse_name(self, img_date):
         imgstrtime = img_date.strftime("%Y-%m-%d %H:00")
@@ -94,6 +120,7 @@ def run(self, now=None):
         self.drop_old_daily_images(now, self.layer_name)
 
         self.update_geonode(self.layer_name, title=self.parse_name(now),
+                            description=self.description,
                             store=self.layer_name,
                             bounds=('-180.0', '180.0',
                                     '-90.0', '90.0', 'EPSG:4326'))

dataqs/forecastio/tasks.py

@@ -1,3 +1,22 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+###############################################################################
+#  Copyright Kitware Inc. and Epidemico Inc.
+#
+#  Licensed under the Apache License, Version 2.0 ( the "License" );
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+###############################################################################
+
 from __future__ import absolute_import
 
 from celery import shared_task

dataqs/forecastio/tests.py

@@ -1,3 +1,22 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+###############################################################################
+#  Copyright Kitware Inc. and Epidemico Inc.
+#
+#  Licensed under the Apache License, Version 2.0 ( the "License" );
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+###############################################################################
+
 import glob
 import zipfile
 import os