Merge pull request #87 from GoogleCloudPlatform/update_compat

updating compat to 0.64

Author: bryanmau1

appengine-compat/exported_appengine_sdk/google/appengine/api/blobstore/blobstore.py

@@ -253,9 +253,9 @@ def create_upload_url(success_path,
     max_bytes_total: The maximum size in bytes that the aggregate sizes of all
       of the blobs in the upload can be or None for no maximum size.
     rpc: Optional UserRPC object.
-    gs_bucket_name: The Google Storage bucket name that the blobs should be
-      uploaded to. The application's service account must have the correct
-      permissions to write to this bucket. The bucket name may be of the foramt
+    gs_bucket_name: The Google Cloud Storage bucket name that the blobs should
+      be uploaded to. The application's service account must have the correct
+      permissions to write to this bucket. The bucket name may be of the format
       'bucket/path/', in which case the included path will be prepended to the
       uploaded object name.
 
@@ -290,9 +290,9 @@ def create_upload_url_async(success_path,
     max_bytes_total: The maximum size in bytes that the aggregate sizes of all
       of the blobs in the upload can be or None for no maximum size.
     rpc: Optional UserRPC object.
-    gs_bucket_name: The Google Storage bucket name that the blobs should be
-      uploaded to. The application's service account must have the correct
-      permissions to write to this bucket. The bucket name may be of the foramt
+    gs_bucket_name: The Google Cloud Storage bucket name that the blobs should
+      be uploaded to. The application's service account must have the correct
+      permissions to write to this bucket. The bucket name may be of the format
       'bucket/path/', in which case the included path will be prepended to the
       uploaded object name.
 

appengine-compat/exported_appengine_sdk/google/appengine/api/datastore.py

@@ -338,6 +338,9 @@ class DatastoreAdapter(datastore_rpc.AbstractAdapter):
           entity_pb.Index_Property.DESCENDING: Index.DESCENDING
       }
 
+  def __init__(self, _id_resolver=None):
+    super(DatastoreAdapter, self).__init__(_id_resolver)
+
   def key_to_pb(self, key):
     return key._Key__reference
 
@@ -384,7 +387,11 @@ def __InitConnection():
 
   if os.getenv(_ENV_KEY) and hasattr(_thread_local, 'connection_stack'):
     return
-  _thread_local.connection_stack = [datastore_rpc.Connection(adapter=_adapter)]
+
+  _thread_local.connection_stack = [
+      datastore_rpc._CreateDefaultConnection(datastore_rpc.Connection,
+                                             adapter=_adapter)]
+
 
   os.environ[_ENV_KEY] = '1'
 

appengine-compat/exported_appengine_sdk/google/appengine/datastore/datastore_rpc.py

@@ -2664,6 +2664,168 @@ def __rollback_hook(self, rpc):
     self.check_rpc_success(rpc)
 
 
+_DATASTORE_APP_ID_ENV = 'DATASTORE_APP_ID'
+_DATASTORE_PROJECT_ID_ENV = 'DATASTORE_PROJECT_ID'
+_DATASTORE_ADDITIONAL_APP_IDS_ENV = 'DATASTORE_ADDITIONAL_APP_IDS'
+_DATASTORE_USE_PROJECT_ID_AS_APP_ID_ENV = 'DATASTORE_USE_PROJECT_ID_AS_APP_ID'
+
+
+
+def _CreateDefaultConnection(connection_fn, **kwargs):
+  """Creates a new connection to Datastore.
+
+  Uses environment variables to determine if the connection should be made
+  to Cloud Datastore v1beta3 or to Datastore's private App Engine API.
+  If DATASTORE_PROJECT_ID exists, connect to Datastore v1beta3. In this case,
+  either DATASTORE_APP_ID or DATASTORE_USE_PROJECT_ID_AS_APP_ID must be set to
+  indicate what the environment's application should be.
+
+  Args:
+    connection_fn: The function to use to create the connection.
+    **kwargs: Addition arguments to pass to the connection_fn.
+
+  Raises:
+    ValueError: If DATASTORE_PROJECT_ID is set but DATASTORE_APP_ID or
+       DATASTORE_USE_PROJECT_ID_AS_APP_ID is not. If DATASTORE_APP_ID doesn't
+       resolve to DATASTORE_PROJECT_ID. If DATASTORE_APP_ID doesn't match
+       an existing APPLICATION_ID.
+
+  Returns:
+    the connection object returned from connection_fn.
+  """
+  datastore_app_id = os.environ.get(_DATASTORE_APP_ID_ENV, None)
+  datastore_project_id = os.environ.get(_DATASTORE_PROJECT_ID_ENV, None)
+  if datastore_app_id or datastore_project_id:
+
+    app_id_override = bool(os.environ.get(
+        _DATASTORE_USE_PROJECT_ID_AS_APP_ID_ENV, False))
+    if not datastore_app_id and not app_id_override:
+      raise ValueError('Could not determine app id. To use project id (%s) '
+                       'instead, set %s=true. This will affect the '
+                       'serialized form of entities and should not be used '
+                       'if serialized entities will be shared between '
+                       'code running on App Engine and code running off '
+                       'App Engine. Alternatively, set %s=<app id>.'
+                       % (datastore_project_id,
+                          _DATASTORE_USE_PROJECT_ID_AS_APP_ID_ENV,
+                          _DATASTORE_APP_ID_ENV))
+    elif datastore_app_id:
+      if app_id_override:
+        raise ValueError('App id was provided (%s) but %s was set to true. '
+                         'Please unset either %s or %s.' %
+                         (datastore_app_id,
+                          _DATASTORE_USE_PROJECT_ID_AS_APP_ID_ENV,
+                          _DATASTORE_APP_ID_ENV,
+                          _DATASTORE_USE_PROJECT_ID_AS_APP_ID_ENV))
+      elif datastore_project_id:
+
+        id_resolver = datastore_pbs.IdResolver([datastore_app_id])
+        if (datastore_project_id !=
+            id_resolver.resolve_project_id(datastore_app_id)):
+          raise ValueError('App id "%s" does not match project id "%s".'
+                           % (datastore_app_id, datastore_project_id))
+
+    datastore_app_id = datastore_app_id or datastore_project_id
+    additional_app_str = os.environ.get(_DATASTORE_ADDITIONAL_APP_IDS_ENV, '')
+    additional_apps = (app.strip() for app in additional_app_str.split(','))
+    return _CreateCloudDatastoreConnection(connection_fn,
+                                           datastore_app_id,
+                                           additional_apps,
+                                           kwargs)
+  return connection_fn(**kwargs)
+
+
+
+def _CreateCloudDatastoreConnection(connection_fn,
+                                    app_id,
+                                    external_app_ids,
+                                    kwargs):
+  """Creates a new context to connect to a remote Cloud Datastore instance.
+
+  This should only be used outside of Google App Engine.
+
+  Args:
+    connection_fn: A connection function which accepts both an _api_version
+      and an _id_resolver argument.
+    app_id: The application id to connect to. This differs from the project
+      id as it may have an additional prefix, e.g. "s~" or "e~".
+    external_app_ids: A list of apps that may be referenced by data in your
+      application. For example, if you are connected to s~my-app and store keys
+      for s~my-other-app, you should include s~my-other-app in the external_apps
+      list.
+    kwargs: The additional kwargs to pass to the connection_fn.
+
+  Raises:
+    ValueError: if the app_id provided doesn't match the current environment's
+        APPLICATION_ID.
+
+  Returns:
+    An ndb.Context that can connect to a Remote Cloud Datastore. You can use
+    this context by passing it to ndb.set_context.
+  """
+
+
+  from google.appengine.datastore import cloud_datastore_v1_remote_stub
+
+  if not datastore_pbs._CLOUD_DATASTORE_ENABLED:
+    raise datastore_errors.BadArgumentError(
+        datastore_pbs.MISSING_CLOUD_DATASTORE_MESSAGE)
+
+  current_app_id = os.environ.get('APPLICATION_ID', None)
+  if current_app_id and current_app_id != app_id:
+
+
+    raise ValueError('Cannot create a Cloud Datastore context that connects '
+                     'to an application (%s) that differs from the application '
+                     'already connected to (%s).' % (app_id, current_app_id))
+  os.environ['APPLICATION_ID'] = app_id
+
+  id_resolver = datastore_pbs.IdResolver((app_id,) + tuple(external_app_ids))
+  project_id = id_resolver.resolve_project_id(app_id)
+  endpoint = googledatastore.helper.get_project_endpoint_from_env(project_id)
+  datastore = googledatastore.Datastore(
+      project_endpoint=endpoint,
+      credentials=googledatastore.helper.get_credentials_from_env())
+  kwargs['_api_version'] = _CLOUD_DATASTORE_V1
+  kwargs['_id_resolver'] = id_resolver
+  conn = connection_fn(**kwargs)
+
+
+
+  try:
+    stub = cloud_datastore_v1_remote_stub.CloudDatastoreV1RemoteStub(datastore)
+    apiproxy_stub_map.apiproxy.RegisterStub(_CLOUD_DATASTORE_V1,
+                                            stub)
+  except:
+    pass
+
+
+
+  try:
+    apiproxy_stub_map.apiproxy.RegisterStub('memcache', _ThrowingStub())
+  except:
+    pass
+  try:
+    apiproxy_stub_map.apiproxy.RegisterStub('taskqueue', _ThrowingStub())
+  except:
+    pass
+
+  return conn
+
+
+class _ThrowingStub(object):
+  """A Stub implementation which always throws a NotImplementedError."""
+
+
+  def MakeSyncCall(self, service, call, request, response):
+    raise NotImplementedError('In order to use %s.%s you must '
+                              'install the Remote API.' % (service, call))
+
+
+  def CreateRPC(self):
+    return apiproxy_rpc.RPC(stub=self)
+
+
 
 
 

appengine-compat/exported_appengine_sdk/google/appengine/ext/vmruntime/callback.py

@@ -28,44 +28,31 @@
 _callback_storage = {}
 
 
-class OnlyOneCallbackPerRequestException(Exception):
-  """Raised when a second callback is set in a single request."""
-
-
 def SetRequestEndCallback(callback):
   """Stores a callback by the request ID.
 
   The request ID currently uses the cloud trace ID.
 
   Args:
     callback: A zero-argument callable whose return value is unused.
-
-  Raises:
-    OnlyOneCallbackPerRequestException: Raised on registering second callback.
   """
   key = _GetRequestId()
 
 
 
 
   if key:
-
-
-
-
-
-    if key not in _callback_storage:
-      _callback_storage[key] = callback
-    else:
-      raise OnlyOneCallbackPerRequestException()
+    _callback_storage.setdefault(key, []).append(callback)
 
 
 def InvokeCallbacks():
-  """Invokes the callback associated with the current request ID."""
+  """Invokes the callbacks associated with the current request ID."""
 
   key = _GetRequestId()
   if key in _callback_storage:
-    _callback_storage[key]()
+    for callback in _callback_storage[key]:
+      callback()
+
     del _callback_storage[key]
 
 

appengine-compat/exported_appengine_sdk/google/appengine/tools/dev-channel-js.js

@@ -1520,6 +1520,9 @@ goog.array.copyByIndex = function(arr, index_arr) {
   });
   return result;
 };
+goog.array.concatMap = function(arr, f, opt_obj) {
+  return goog.array.concat.apply([], goog.array.map(arr, f, opt_obj));
+};
 goog.dom.TagName = {A:"A", ABBR:"ABBR", ACRONYM:"ACRONYM", ADDRESS:"ADDRESS", APPLET:"APPLET", AREA:"AREA", ARTICLE:"ARTICLE", ASIDE:"ASIDE", AUDIO:"AUDIO", B:"B", BASE:"BASE", BASEFONT:"BASEFONT", BDI:"BDI", BDO:"BDO", BIG:"BIG", BLOCKQUOTE:"BLOCKQUOTE", BODY:"BODY", BR:"BR", BUTTON:"BUTTON", CANVAS:"CANVAS", CAPTION:"CAPTION", CENTER:"CENTER", CITE:"CITE", CODE:"CODE", COL:"COL", COLGROUP:"COLGROUP", COMMAND:"COMMAND", DATA:"DATA", DATALIST:"DATALIST", DD:"DD", DEL:"DEL", DETAILS:"DETAILS", DFN:"DFN", 
 DIALOG:"DIALOG", DIR:"DIR", DIV:"DIV", DL:"DL", DT:"DT", EM:"EM", EMBED:"EMBED", FIELDSET:"FIELDSET", FIGCAPTION:"FIGCAPTION", FIGURE:"FIGURE", FONT:"FONT", FOOTER:"FOOTER", FORM:"FORM", FRAME:"FRAME", FRAMESET:"FRAMESET", H1:"H1", H2:"H2", H3:"H3", H4:"H4", H5:"H5", H6:"H6", HEAD:"HEAD", HEADER:"HEADER", HGROUP:"HGROUP", HR:"HR", HTML:"HTML", I:"I", IFRAME:"IFRAME", IMG:"IMG", INPUT:"INPUT", INS:"INS", ISINDEX:"ISINDEX", KBD:"KBD", KEYGEN:"KEYGEN", LABEL:"LABEL", LEGEND:"LEGEND", LI:"LI", LINK:"LINK", 
 MAP:"MAP", MARK:"MARK", MATH:"MATH", MENU:"MENU", META:"META", METER:"METER", NAV:"NAV", NOFRAMES:"NOFRAMES", NOSCRIPT:"NOSCRIPT", OBJECT:"OBJECT", OL:"OL", OPTGROUP:"OPTGROUP", OPTION:"OPTION", OUTPUT:"OUTPUT", P:"P", PARAM:"PARAM", PRE:"PRE", PROGRESS:"PROGRESS", Q:"Q", RP:"RP", RT:"RT", RUBY:"RUBY", S:"S", SAMP:"SAMP", SCRIPT:"SCRIPT", SECTION:"SECTION", SELECT:"SELECT", SMALL:"SMALL", SOURCE:"SOURCE", SPAN:"SPAN", STRIKE:"STRIKE", STRONG:"STRONG", STYLE:"STYLE", SUB:"SUB", SUMMARY:"SUMMARY",