Skip to content
This repository was archived by the owner on Jan 12, 2022. It is now read-only.

Commit df93686

Browse files
andrewsgandrewsg
committed
Monkey-patch os.env to be threadlocal and set up environment to mimic GAE
1 parent 06fede9 commit df93686

4 files changed

Lines changed: 391 additions & 25 deletions

File tree

‎multicore_runtime/middleware.py‎

Lines changed: 162 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,162 @@
1+
# Copyright 2015 Google Inc. All Rights Reserved.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
#
15+
"""WSGI middleware to wrap the dispatcher, and supporting functions."""
16+
17+
import logging
18+
import os
19+
20+
21+
RESERVED_ENV_KEYS = {
22+
'AUTH_DOMAIN': 'gmail.com', # Default auth domain must be set.
23+
'DATACENTER': '',
24+
'DEFAULT_VERSION_HOSTNAME': '',
25+
'HTTPS': '',
26+
'REMOTE_ADDR': '',
27+
'REQUEST_ID_HASH': '',
28+
'REQUEST_LOG_ID': '',
29+
'USER_EMAIL': '',
30+
'USER_ID': '',
31+
'USER_IS_ADMIN': '0', # Default admin flag to explicit '0'.
32+
'USER_NICKNAME': '',
33+
'USER_ORGANIZATION': '',}
34+
35+
36+
def reset_environment_middleware(app, frozen_environment, frozen_user_env,
37+
frozen_appengine_config_env):
38+
"""Replace the contents of os.environ with a frozen env + request data.
39+
40+
This requires a single-threaded webserver, or for os.environ to be patched to
41+
be thread-local.
42+
43+
Args:
44+
app: The WSGI app to wrap.
45+
frozen_environment: An iterable of (key, value) tuples that can be used to
46+
populate os.environ with the initial state of the environment.
47+
`tuple(os.environ.iteritems())` produces appropriate output.
48+
frozen_user_env: An iterable of (key, value) tuples that can be used to
49+
populate os.environ with environment variables specified in app.yaml.
50+
frozen_appengine_config_env: An iterable of (key, value) tuples that can be
51+
used to populate os.environ with configuration-dependent env variables.
52+
53+
Returns:
54+
The wrapped app, also a WSGI app.
55+
"""
56+
57+
def reset_environment_wrapper(wsgi_env, start_response): # pylint: disable=missing-docstring
58+
# Wipe os.environ entirely.
59+
os.environ.clear()
60+
61+
# Populate os.environ in order, so that later steps overwrite conflicting
62+
# keys in previous steps.
63+
64+
# Add in user env variables specified in app.yaml. These are added first
65+
# because they should not take precedence over any conflicting key.
66+
os.environ.update(frozen_user_env)
67+
68+
# Repopulate os.environ with the frozen environment.
69+
os.environ.update(frozen_environment)
70+
71+
# Add in wsgi_env data, including request headers.
72+
os.environ.update(request_environment_for_wsgi_env(wsgi_env))
73+
74+
# Add in configuration data from appengine_config.
75+
os.environ.update(frozen_appengine_config_env)
76+
77+
# Add reserved keys, which draw from wsgi_env as well. These have a very
78+
# high priority and so are added nearly last.
79+
os.environ.update(reserved_env_keys_for_wsgi_env(wsgi_env))
80+
81+
# Tweak the environment to hide the service bridge.
82+
os.environ.update(get_env_to_hide_service_bridge(wsgi_env))
83+
84+
return app(wsgi_env, start_response)
85+
86+
return reset_environment_wrapper
87+
88+
89+
def request_environment_for_wsgi_env(wsgi_env):
90+
"""Get a dict of key-value pairs from wsgi_env that work as env variables.
91+
92+
Does not mutate input.
93+
94+
Args:
95+
wsgi_env: WSGI request env data.
96+
97+
Returns:
98+
A dictionary suitable for use in `os.environ.update(output)`.
99+
"""
100+
101+
# Return all key, value pairs from wsgi_env where value is a string.
102+
return {key: value for key, value in wsgi_env.iteritems()
103+
if isinstance(value, basestring)}
104+
105+
106+
def reserved_env_keys_for_wsgi_env(wsgi_env):
107+
"""Get a dict for reserved keys based on wsgi_env headers and defaults.
108+
109+
Does not mutate input.
110+
111+
Args:
112+
wsgi_env: WSGI request env data.
113+
114+
Returns:
115+
A dictionary suitable for use in `os.environ.update(output)`.
116+
"""
117+
118+
output = {}
119+
120+
# Use the default value for a reserved key if the corresponding header is not
121+
# set, or if the header exists but its value is blank.
122+
for key, default in RESERVED_ENV_KEYS.iteritems():
123+
value = wsgi_env.get('HTTP_X_APPENGINE_' + key)
124+
output[key] = value or default # Must be set to a valid value or default.
125+
126+
return output
127+
128+
129+
def get_env_to_hide_service_bridge(wsgi_env):
130+
"""Generate a dictionary of environment variables to hide the service bridge.
131+
132+
Does not mutate input.
133+
134+
Args:
135+
wsgi_env: WSGI request env data.
136+
137+
Returns:
138+
A dictionary suitable for use in `os.environ.update(output)`.
139+
"""
140+
output = {}
141+
142+
# Because the request is coming over the service bridge, the service bridge
143+
# host and port that are automatically populated in SERVER_NAME and
144+
# SERVER_PORT, respectively. However, this is an implementation detail that
145+
# should not be shown to user code. Instead, we'll rely on the HTTP Host
146+
# header to retrieve the hostname used in the original request. This mimics
147+
# the behavior of non-VM App Engine.
148+
if 'HTTP_HOST' in wsgi_env:
149+
output['SERVER_NAME'] = wsgi_env['HTTP_HOST']
150+
151+
# Similarly we'll use the HTTPS flag to determine the port used in the
152+
# original request.
153+
https = wsgi_env.get('HTTP_X_APPENGINE_HTTPS', 'off')
154+
if https == 'off':
155+
output['SERVER_PORT'] = '80'
156+
elif https == 'on':
157+
output['SERVER_PORT'] = '443'
158+
else:
159+
logging.warning('Unrecognized value for HTTPS (%s), won\'t modify '
160+
'SERVER_PORT', https)
161+
162+
return output

‎multicore_runtime/wsgi.py‎

Lines changed: 40 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -15,19 +15,23 @@
1515
"""Configure a user project and instantiate WSGI app meta_app to serve it.
1616
1717
Importing this module will result in side-effects, such as registering the
18-
project's default ticket via vmstub.Register. This is a straightforward and
19-
compatible way to induce the webserver to run initialization code before
20-
starting to serve the WSGI app.
18+
project's default ticket via vmstub.Register. This is a broadly compatible way
19+
to induce the webserver to run initialization code before starting to serve the
20+
WSGI app.
2121
"""
2222

2323
import logging
24+
import os
2425

2526
from dispatcher import dispatcher
26-
from wsgi_utils import get_module_config
27-
from wsgi_utils import get_module_config_filename
28-
from wsgi_utils import load_user_scripts_into_handlers
27+
from middleware import reset_environment_middleware
28+
from wsgi_config import env_vars_from_appengine_config
29+
from wsgi_config import get_module_config
30+
from wsgi_config import get_module_config_filename
31+
from wsgi_config import load_user_scripts_into_handlers
32+
from wsgi_config import ThreadLocalDict
33+
from wsgi_config import user_env_vars_from_appinfo_external
2934

30-
from google.appengine.ext.vmruntime import middlewares
3135
from google.appengine.ext.vmruntime import vmconfig
3236
from google.appengine.ext.vmruntime import vmstub
3337

@@ -39,14 +43,38 @@
3943
# Ensure API requests include a valid ticket by default.
4044
vmstub.Register(vmstub.VMStub(appengine_config.default_ticket))
4145

42-
# Load user code
46+
# Load user code.
4347
preloaded_handlers = load_user_scripts_into_handlers(appinfo_external.handlers)
4448

4549
# Now that all scripts are fully imported, it is safe to use asynchronous
4650
# API calls.
4751
# TODO(apphosting): change this to use an env variable instead of module state
4852
vmstub.app_is_loaded = True
4953

54+
# Take an immutable snapshot of the environment's current state, which we will
55+
# use to refresh the environment (via `reset_environment_middleware`) at the
56+
# beginning of each request.
57+
frozen_environment = tuple(os.environ.iteritems())
58+
59+
# Also freeze user env vars specified in app.yaml. Later steps to modify the
60+
# environment such as env_vars_from_appengine_config and request middleware
61+
# will overwrite these changes. This is added to the environment in
62+
# `reset_environment_middleware`.
63+
frozen_user_env = tuple(
64+
user_env_vars_from_appinfo_external(appinfo_external).iteritems())
65+
66+
# Also freeze environment data from appengine_config. This is added to the
67+
# environment in `reset_environment_middleware`.
68+
frozen_appengine_config_env = tuple(
69+
env_vars_from_appengine_config(appengine_config).iteritems())
70+
71+
# Monkey-patch os.environ to be thread-local. This is for backwards
72+
# compatibility with GAE's use of environment variables to store request data.
73+
# Note: gunicorn "gevent" or "eventlet" workers, if selected, will
74+
# automatically monkey-patch the threading module to make this work with green
75+
# threads.
76+
os.environ = ThreadLocalDict()
77+
5078
# Create a "meta app" that dispatches requests based on handlers.
5179
meta_app = dispatcher(preloaded_handlers)
5280

@@ -55,13 +83,7 @@
5583
# layer (the middleware code that will process a request first). Inside the
5684
# innermost layer is the actual dispatcher, above.
5785

58-
# Adjust SERVER_NAME and SERVER_PORT env vars to hide the service bridge.
59-
meta_app = middlewares.FixServerEnvVarsMiddleware(meta_app)
60-
61-
# Patch os.environ to be thread local, and stamp it with default values based
62-
# on the app configuration. This is for backwards compatibility with GAE's use
63-
# of environment variables to store request data.
64-
# Note: gunicorn "gevent" or "eventlet" workers, if selected, will
65-
# automatically monkey-patch the threading module to make this compatible with
66-
# green threads.
67-
meta_app = middlewares.OsEnvSetupMiddleware(meta_app, appengine_config)
86+
# Reset os.environ to the frozen state and add request-specific data.
87+
meta_app = reset_environment_middleware(meta_app, frozen_environment,
88+
frozen_user_env,
89+
frozen_appengine_config_env)

multicore_runtime/wsgi_utils.py renamed to multicore_runtime/wsgi_config.py

Lines changed: 49 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,10 +12,12 @@
1212
# See the License for the specific language governing permissions and
1313
# limitations under the License.
1414
#
15-
"""Utilities to configure the dispatcher."""
15+
"""Utilities to configure the dispatcher, middleware and environment."""
1616

1717
import logging
1818
import os
19+
import threading
20+
from UserDict import UserDict
1921

2022
from google.appengine.api import appinfo_includes
2123
from google.appengine.runtime import wsgi
@@ -67,3 +69,49 @@ def load_user_scripts_into_handlers(handlers):
6769
logging.info('Parsed handlers: %s',
6870
[(url, script) for (url, script, _) in loaded_handlers])
6971
return loaded_handlers
72+
73+
74+
def env_vars_from_appengine_config(appengine_config):
75+
"""Generate a dict suitable for updating os.environ to reflect app config.
76+
77+
This function only returns a dict and does not update os.environ directly.
78+
79+
Args:
80+
appengine_config: The app configuration as generated by
81+
vmconfig.BuildVmAppengineEnvConfig()
82+
83+
Returns:
84+
A dict of strings suitable for e.g. `os.environ.update(values)`.
85+
"""
86+
87+
return {'SERVER_SOFTWARE': appengine_config.server_software,
88+
'APPENGINE_RUNTIME': 'python27',
89+
'APPLICATION_ID': '%s~%s' % (appengine_config.partition,
90+
appengine_config.appid),
91+
'INSTANCE_ID': appengine_config.instance,
92+
'BACKEND_ID': appengine_config.major_version,
93+
'CURRENT_MODULE_ID': appengine_config.module,
94+
'CURRENT_VERSION_ID': '%s.%s' % (appengine_config.major_version,
95+
appengine_config.minor_version),
96+
'DEFAULT_TICKET': appengine_config.default_ticket}
97+
98+
99+
def user_env_vars_from_appinfo_external(appinfo_external):
100+
"""Generate a dict of env variables specified by the user in app.yaml.
101+
102+
This function only returns a dict and does not update os.environ directly.
103+
104+
Args:
105+
appinfo_external: The app.yaml configuration info as generated by
106+
get_module_config()
107+
108+
Returns:
109+
A dict of strings suitable for e.g. `os.environ.update(values)`.
110+
"""
111+
112+
return appinfo_external.env_variables or {}
113+
114+
115+
# Dictionary with thread-local contents.
116+
class ThreadLocalDict(UserDict, threading.local):
117+
pass

0 commit comments

Comments
 (0)