static_files and static_dir handling and tests

Author: andrewsg

multicore_runtime/dispatcher.py

@@ -18,6 +18,7 @@
 import re
 
 from werkzeug.wrappers import Request
+from werkzeug.wrappers import Response
 
 
 def dispatcher(handlers):
@@ -32,29 +33,21 @@ def dispatcher(handlers):
     A WSGI app that dispatches to the user apps specified in the input.
   """
 
-  def dispatch(env, start_response):
+  def dispatch(wsgi_env, start_response):
     """Handle one request."""
-    request = Request(env)
-    for url, script, app in handlers:  # Closure over parent func's handlers.
+    request = Request(wsgi_env)
+    for url, script, app in handlers:  # pylint: disable=unused-variable
       matcher = re.match(url, request.path)
       if matcher and matcher.end() == len(request.path):
-        if script:
-          if app is not None:
-            # Send a response via the app specified in the handler.
-            return app(env, start_response)
-          else:
-            # The import must have failed. This will have been logged at import
-            # time. Send a 500 error response.
-            start_response('500 Internal Server Error', [])
-            return ['<h1>500 Internal Server Error</h1>\n']
-        # This is a static file; normally these are handled by the appserver
-        # but in development or for local queries they may end up here.
-        # TODO(apphosting): support static files.
+        if app is not None:
+          # Send a response via the app specified in the handler.
+          return app(wsgi_env, start_response)
         else:
-          logging.error('Static file serving is not supported: %s ',
-                        request.path)
-          start_response('500 Internal Server Error', [])
-          return ['<h1>500 Internal Server Error</h1>\n']
+          # The import must have failed. This will have been logged at import
+          # time. Send a 500 error response.
+          response = Response('<h1>500 Internal Server Error</h1>\n',
+                              status=500)
+          return response(wsgi_env, start_response)
     logging.error('No handler found for %s', request.path)
     start_response('404 Not Found', [])
     return ['<h1>404 Not Found</h1>\n']

multicore_runtime/static_files.py

@@ -0,0 +1,69 @@
+# Copyright 2015 Google Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+"""Static file serving functionality invoked by wsgi_config.py."""
+
+import logging
+import mimetypes
+import os
+import re
+
+from werkzeug.wrappers import Request
+from werkzeug.wrappers import Response
+from werkzeug.wsgi import wrap_file
+
+
+def static_app_for_regex_and_files(regex, files, mime_type=None):
+  """Returns a WSGI app that serves static files.
+
+  Args:
+    regex: A url-matching regex as specified in appinfo.URLMap.
+    files: A static_files definition as specified in appinfo.URLMap.
+        May include a regex backref. See the appinfo.URLMap docstring
+        for more information.
+    mime_type: A mime type to apply to all files. If absent,
+        mimetypes.guess_type() is used.
+
+  Returns:
+    A static file-serving WSGI app closed over the inputs.
+  """
+  @Request.application  # Transforms wsgi_env, start_response args into request
+  def serve_static_files(request):
+    """Serve a static file."""
+    # First, match the path against the regex...
+    matcher = re.match(regex, request.path)
+    # ... and use the files regex backref to choose a filename.
+    filename = matcher.expand(files)
+
+    # Rudimentary protection against path traversal outside of the app. This
+    # code should never be hit in production, as Google's frontend servers
+    # (GFE) rewrite traversal attempts and respond with a 302 immediately.
+    filename = os.path.abspath(filename)
+    if not filename.startswith(os.path.join(os.getcwd(), '')):
+      logging.warn('Path traversal protection triggered for %s, '
+                   'returning 404', filename)
+      return Response(status=404)
+
+    try:
+      fp = open(filename, 'rb')
+      # fp is not closed in this function as it is handed to the WSGI server
+      # directly.
+    except IOError:
+      logging.warn('Requested non-existent filename %s', filename)
+      return Response(status=404)
+
+    wrapped_file = wrap_file(request.environ, fp)
+    return Response(wrapped_file, direct_passthrough=True,
+                    mimetype=mime_type or mimetypes.guess_type(filename)[0])
+  return serve_static_files

multicore_runtime/test_statics/favicon.ico

@@ -0,0 +1 @@
+favicon.ico contents

multicore_runtime/wsgi.py

@@ -24,8 +24,8 @@
 import os
 
 from dispatcher import dispatcher
-from middleware import reset_environment_middleware
 from middleware import health_check_middleware
+from middleware import reset_environment_middleware
 from wsgi_config import env_vars_from_env_config
 from wsgi_config import get_module_config
 from wsgi_config import get_module_config_filename
@@ -91,4 +91,4 @@
 # Reset os.environ to the frozen state and add request-specific data.
 meta_app = reset_environment_middleware(meta_app, frozen_environment,
                                         frozen_user_env,
-                                        frozen_env_config_env)
+                                        frozen_env_config_env)

multicore_runtime/wsgi_config.py

@@ -19,6 +19,8 @@
 import threading
 from UserDict import UserDict
 
+from static_files import static_app_for_regex_and_files
+
 from google.appengine.api import appinfo_includes
 from google.appengine.runtime import wsgi
 
@@ -38,14 +40,13 @@ def get_module_config(filename):
 
 def app_for_script(script):
   """Returns the WSGI app specified in the input string, or None on failure."""
-  if script:
-    app, filename, err = wsgi.LoadObject(script)  # pylint: disable=unused-variable
-    if err:
-      # Log the exception but do not reraise.
-      logging.exception('Failed to import %s: %s', script, err)
-      return None
-    else:
-      return app_wrapped_in_user_middleware(app)
+  app, filename, err = wsgi.LoadObject(script)  # pylint: disable=unused-variable
+  if err:
+    # Log the exception but do not reraise.
+    logging.exception('Failed to import %s: %s', script, err)
+    return None
+  else:
+    return app_wrapped_in_user_middleware(app)
 
 
 def app_wrapped_in_user_middleware(app):
@@ -80,6 +81,51 @@ def get_add_middleware_from_appengine_config():
     return None
 
 
+def static_app_for_handler(handler):
+  """Returns a WSGI app that serves static files as directed by the handler.
+
+  Args:
+    handler: An individual handler from appinfo_external.handlers
+      (appinfo.URLMap)
+
+  Returns:
+    A static file-serving WSGI app closed over the handler information.
+  """
+  regex = handler.url
+  files = handler.static_files
+  if not files:
+    if handler.static_dir:
+      # If static_files is not set, convert static_dir to static_files and also
+      # modify the url regex accordingly. See the appinfo.URLMap docstring for
+      # more information.
+      regex = static_dir_url(handler)
+      files = handler.static_dir + r'/\1'
+    else:
+      # Neither static_files nor static_dir is set; log an error and return.
+      logging.error('No script, static_files or static_dir found for %s',
+                    handler)
+      return None
+  return static_app_for_regex_and_files(regex, files,
+                                        mime_type=handler.mime_type)
+
+
+def static_dir_url(handler):
+  """Converts a static_dir regex into a static_files regex if needed.
+
+  See the appinfo.URLMap docstring for more information.
+
+  Args:
+    handler: A handler (appinfo.URLMap)
+
+  Returns:
+    A modified url regex
+  """
+  if not handler.script and not handler.static_files and handler.static_dir:
+    return handler.url + '/(.*)'
+  else:
+    return handler.url
+
+
 def load_user_scripts_into_handlers(handlers):
   """Preloads user scripts, wrapped in env_config middleware if present.
 
@@ -94,9 +140,9 @@ def load_user_scripts_into_handlers(handlers):
       - app: The fully loaded app corresponding to the script.
   """
   loaded_handlers = [
-      (x.url,
+      (x.url if x.script or x.static_files else static_dir_url(x),
        x.script.replace('$PYTHON_LIB/', '') if x.script else x.script,
-       app_for_script(x.script) if x.script else None)
+       app_for_script(x.script) if x.script else static_app_for_handler(x))
       for x in handlers]
   logging.info('Parsed handlers: %s',
                [(url, script) for (url, script, _) in loaded_handlers])

multicore_runtime/wsgi_test.py

@@ -34,7 +34,15 @@
     appinfo.URLMap(url='/failure', script='wsgi_test.nonexistent_function'),
     appinfo.URLMap(url='/env', script='wsgi_test.dump_os_environ'),
     appinfo.URLMap(url='/setenv', script='wsgi_test.add_to_os_environ'),
-    appinfo.URLMap(url='/wait', script='wsgi_test.wait_on_global_event')]
+    appinfo.URLMap(url='/wait', script='wsgi_test.wait_on_global_event'),
+    appinfo.URLMap(url='/favicon.ico', static_files='test_statics/favicon.ico'),
+    appinfo.URLMap(url='/faketype.ico', static_files='test_statics/favicon.ico',
+                   mime_type='application/fake_type'),
+    appinfo.URLMap(url='/wildcard_statics/(.*)',
+                   static_files=r'test_statics/\1'),
+    appinfo.URLMap(url='/static_dir',
+                   static_dir='test_statics'),
+    ]
 HELLO_STRING = 'Hello World!'
 
 FAKE_ENV_KEY = 'KEY'
@@ -118,6 +126,10 @@ def test_notfound(self):
     response = self.client.get('/notfound')
     self.assertEqual(response.status_code, 404)
 
+  def test_health(self):
+    response = self.client.get('/_ah/health')
+    self.assertEqual(response.status_code, 200)
+
   # Test PATH is present in env. If this breaks, each request is properly
   # wiping the environment but not properly reconstituting the frozen initial
   # state.
@@ -126,6 +138,39 @@ def test_basic_env(self):
     # Assumes PATH will be present in the env in all cases, including tests!
     self.assertIn('PATH', json.loads(response.data))
 
+  def test_static_file(self):
+    response = self.client.get('/favicon.ico')
+    self.assertEqual(response.status_code, 200)
+    with open('test_statics/favicon.ico') as f:
+      self.assertEqual(response.data, f.read())
+
+  def test_static_file_mime_type(self):
+    response = self.client.get('/faketype.ico')
+    self.assertEqual(response.status_code, 200)
+    with open('test_statics/favicon.ico') as f:
+      self.assertEqual(response.data, f.read())
+    self.assertEqual(response.mimetype, 'application/fake_type')
+
+  def test_static_file_wildcard(self):
+    response = self.client.get('/wildcard_statics/favicon.ico')
+    self.assertEqual(response.status_code, 200)
+    with open('test_statics/favicon.ico') as f:
+      self.assertEqual(response.data, f.read())
+
+  def test_static_file_wildcard_404(self):
+    response = self.client.get('/wildcard_statics/no_file')
+    self.assertEqual(response.status_code, 404)
+
+  def test_static_file_wildcard_directory_traversal(self):
+    response = self.client.get('/wildcard_statics/../../setup.py')
+    self.assertEqual(response.status_code, 404)
+
+  def test_static_dir(self):
+    response = self.client.get('/static_dir/favicon.ico')
+    self.assertEqual(response.status_code, 200)
+    with open('test_statics/favicon.ico') as f:
+      self.assertEqual(response.data, f.read())
+
   def test_wsgi_vars_in_env(self):
     response = self.client.get('/env')
     env = json.loads(response.data)