multicore runtime with dispatcher and tests

Author: andrewsg

multicore_runtime/base_requirements.txt

@@ -0,0 +1,15 @@
+# Multicore web application server.
+gunicorn
+
+# Better HTTP api calls, used by vmstub.py.
+requests
+
+# Required by the base runtime.
+webapp2
+webob
+
+# Required to parse app.yaml.
+pyyaml
+
+# Better request/response processing for the meta app.
+werkzeug

multicore_runtime/dev/Dockerfile

@@ -0,0 +1,28 @@
+# Base Dockerfile for all AppEngine runtime implementations.
+FROM google/debian:wheezy
+# Clean any CMD that might be inherited from previous image, because that
+# will pollute our ENTRYPOINT, see
+# http://docs.docker.io/en/latest/reference/builder/#entrypoint.
+CMD []
+ENV DEBIAN_FRONTEND noninteractive
+RUN apt-get -q update ; \
+  apt-get install --no-install-recommends -y -q ca-certificates net-tools lsof && \
+  apt-get -y -q upgrade
+
+RUN apt-get -q update ; apt-get -y -q --no-install-recommends install python2.7 python-pip
+
+ADD python_vm_runtime.tar.gz /home/vmagent/python_vm_runtime
+
+EXPOSE 8080
+
+RUN ln -s /home/vmagent/app /app
+WORKDIR /app
+
+RUN ln -s /home/vmagent/python_vm_runtime/google /home/vmagent/app/google
+
+ENTRYPOINT ["/usr/bin/env", "gunicorn", "-b", "0.0.0.0:8080", "wsgi:meta_app", "--log-file=-"]
+
+ADD . /home/vmagent/app/
+
+RUN pip install -r base_requirements.txt
+

multicore_runtime/dispatcher.py

@@ -0,0 +1,62 @@
+# Copyright 2015 Google Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+"""A WSGI app that, once configured, dispatches requests to user apps."""
+
+import logging
+import re
+
+from werkzeug.wrappers import Request
+
+
+def dispatcher(handlers):
+  """Accepts handlers and returns a WSGI app that dispatches requests to them.
+
+  Args:
+    handlers: a list of handlers as produced by
+      wsgi_utils.load_user_scripts_into_handlers: a list of tuples of
+      (url, script, app).
+
+  Returns:
+    A WSGI app that dispatches to the user apps specified in the input.
+  """
+
+  def dispatch(env, start_response):
+    """Handle one request."""
+    request = Request(env)
+    for url, script, app in handlers:  # Closure over parent func's handlers.
+      matcher = re.match(url, request.path)
+      if matcher and matcher.end() == len(request.path):
+        if script:
+          if app is not None:
+            # Send a response via the app specified in the handler.
+            return app(env, start_response)
+          else:
+            # The import must have failed. This will have been logged at import
+            # time. Send a 500 error response.
+            start_response('500 Internal Server Error', [])
+            return ['<h1>500 Internal Server Error</h1>\n']
+        # This is a static file; normally these are handled by the appserver
+        # but in development or for local queries they may end up here.
+        # TODO(apphosting): support static files.
+        else:
+          logging.error('Static file serving is not supported: %s ',
+                        request.path)
+          start_response('500 Internal Server Error', [])
+          return ['<h1>500 Internal Server Error</h1>\n']
+    logging.error('No handler found for %s', request.path)
+    start_response('404 Not Found', [])
+    return ['<h1>404 Not Found</h1>\n']
+
+  return dispatch

multicore_runtime/google

@@ -0,0 +1 @@
+../python_vm_runtime/google

multicore_runtime/wsgi.py

@@ -0,0 +1,51 @@
+# Copyright 2015 Google Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+"""Configure a user project and instantiate WSGI app meta_app to serve it.
+
+Importing this module will result in side-effects, such as registering the
+project's default ticket via vmstub.Register. This is a straightforward and
+compatible way to induce the webserver to run initialization code before
+starting to serve the WSGI app.
+"""
+
+import logging
+
+from dispatcher import dispatcher
+from wsgi_utils import get_module_config
+from wsgi_utils import get_module_config_filename
+from wsgi_utils import load_user_scripts_into_handlers
+
+from google.appengine.ext.vmruntime import vmconfig
+from google.appengine.ext.vmruntime import vmstub
+
+
+logging.basicConfig(level=logging.INFO)
+
+appinfo_external = get_module_config(get_module_config_filename())
+appengine_config = vmconfig.BuildVmAppengineEnvConfig()
+
+# Ensure API requests include a valid ticket by default.
+vmstub.Register(vmstub.VMStub(appengine_config.default_ticket))
+
+# Load user code
+preloaded_handlers = load_user_scripts_into_handlers(appinfo_external.handlers)
+
+# Now that all scripts are fully imported, it is safe to use asynchronous
+# API calls.
+# TODO(apphosting): change this to use an env variable instead of module state
+vmstub.app_is_loaded = True
+
+# Create a "meta app" that dispatches requests based on handlers.
+meta_app = dispatcher(preloaded_handlers)

multicore_runtime/wsgi_test.py

@@ -0,0 +1,64 @@
+# Copyright 2015 Google Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# pylint: disable=g-import-not-at-top
+
+import unittest
+from mock import MagicMock
+from mock import patch
+
+from werkzeug.test import Client
+from werkzeug.wrappers import Request
+from werkzeug.wrappers import Response
+
+from google.appengine.api import appinfo
+
+FAKE_HANDLERS = [
+    appinfo.URLMap(url='/hello', script='wsgi_test.hello_world'),
+    appinfo.URLMap(url='/failure', script='wsgi_test.nonexistent_function')]
+HELLO_STRING = 'Hello World!'
+
+
+@Request.application
+def hello_world(request):  # pylint: disable=unused-argument
+  return Response(HELLO_STRING)
+
+
+class MetaAppTestCase(unittest.TestCase):
+
+  def setUp(self):
+    # Pre-import modules to patch them in advance.
+    from google.appengine.ext.vmruntime import vmconfig  # pylint: disable=unused-variable
+
+    # Instantiate an app with a simple fake configuration.
+    with patch('wsgi_utils.get_module_config_filename'):
+      with patch('wsgi_utils.get_module_config',
+                 return_value=MagicMock(handlers=FAKE_HANDLERS)):
+        with patch('google.appengine.ext.vmruntime.vmconfig.BuildVmAppengineEnvConfig'):  # pylint: disable=line-too-long
+          import wsgi
+          self.app = wsgi.meta_app
+    self.client = Client(self.app, Response)
+
+  def test_hello(self):
+    response = self.client.get('/hello')
+    self.assertEqual(response.status_code, 200)
+    self.assertEqual(response.data, HELLO_STRING)
+
+  def test_failure(self):
+    response = self.client.get('/failure')
+    self.assertEqual(response.status_code, 500)
+
+  def test_notfound(self):
+    response = self.client.get('/notfound')
+    self.assertEqual(response.status_code, 404)

multicore_runtime/wsgi_utils.py

@@ -0,0 +1,69 @@
+# Copyright 2015 Google Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+"""Utilities to configure the dispatcher."""
+
+import logging
+import os
+
+from google.appengine.api import appinfo_includes
+from google.appengine.runtime import wsgi
+
+
+def get_module_config_filename():
+  """Returns the name of the module configuration file (app.yaml)."""
+  module_yaml_path = os.environ['MODULE_YAML_PATH']
+  logging.info('Using module_yaml_path from env: %s', module_yaml_path)
+  return module_yaml_path
+
+
+def get_module_config(filename):
+  """Returns the parsed module config."""
+  with open(filename) as f:
+    return appinfo_includes.Parse(f)
+
+
+def app_for_script(script):
+  """Returns the WSGI app specified in the input string, or None on failure."""
+  if script:
+    app, filename, err = wsgi.LoadObject(script)  # pylint: disable=unused-variable
+    if err:
+      # Log the exception but do not reraise.
+      logging.exception('Failed to import %s: %s', script, err)
+      return None
+    else:
+      return app
+
+
+def load_user_scripts_into_handlers(handlers):
+  """Preloads user scripts.
+
+  Args:
+    handlers: appinfo_external.handlers data as provided by get_module_config()
+
+  Returns:
+    A list of tuples suitable for configuring the dispatcher() app,
+    where the tuples are (url, script, app):
+      - url: The url pattern which matches this handler.
+      - script: The script to serve for this handler, as a string.
+      - app: The fully loaded app corresponding to the script.
+  """
+  loaded_handlers = [
+      (x.url,
+       x.script.replace('$PYTHON_LIB/', '') if x.script else x.script,
+       app_for_script(x.script) if x.script else None)
+      for x in handlers]
+  logging.info('Parsed handlers: %s',
+               [(url, script) for (url, script, _) in loaded_handlers])
+  return loaded_handlers