Modifying test harness for 10x test development velocity. Client now uploads new test files to cloud storage per test and the test app automatically pulls them down for testing. This bypasses deployment.

Clear test files on Google Cloud Storage and clear files in the
application's test directory.

Author: Bryan Mau

.gitignore

@@ -52,3 +52,8 @@ docs/_build/
 
 # PyBuilder
 target/
+
+# Generated test files
+tests/e2e/app/app.yaml
+tests/e2e/app/Dockerfile
+

common.mk

@@ -4,3 +4,4 @@ ifeq ($(origin DOCKER_IMAGE_NAME), undefined)
 	GCLOUD_PROJECT:=$(shell gcloud config list project --format="value(core.project)")
 	DOCKER_IMAGE_NAME:=gcr.io/$(GCLOUD_PROJECT)/google-python-compat
 endif
+BUCKET=gs://${GCLOUD_PROJECT}-python-compat-e2e-tests

tests/e2e/app/Makefile

@@ -4,7 +4,7 @@ include ../../../common.mk
 all: deploy
 
 .PHONY: deploy
-deploy: build-base generate-dockerfile copy-tests
+deploy: build-base generate-dockerfile generate-app-yaml
 	gcloud preview app deploy -q
 
 .PHONY: build-base
@@ -15,6 +15,6 @@ build-base:
 generate-dockerfile:
 	sed "s#\$$DOCKER_IMAGE_NAME#$(DOCKER_IMAGE_NAME)#g" Dockerfile.in > Dockerfile
 
-.PHONY: copy-tests
-copy-tests:
-	cp -r ../tests .
+.PHONY: generate-app-yaml
+generate-app-yaml:
+	sed "s#\$$CLOUD_STORAGE_BUCKET#$(BUCKET)#g" app.yaml.in > app.yaml

tests/e2e/app/app.yaml tests/e2e/app/app.yaml.intests/e2e/app/app.yaml renamed to tests/e2e/app/app.yaml.in

@@ -4,3 +4,6 @@ vm: true
 handlers:
 - url: .*
   script: main.app
+
+env_variables:
+  CLOUD_STORAGE_BUCKET: $CLOUD_STORAGE_BUCKET

tests/e2e/app/main.py

@@ -12,12 +12,19 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import sys
-import contextlib
 from cStringIO import StringIO
+from gcloud import storage
 
-import webapp2
+import contextlib
 import pytest
+import os
+import shutil
+import sys
+import webapp2
+
+# Configure this environment variable via app.yaml.in.
+# Cut off the 'gs://' portion of the name.
+GCLOUD_STORAGE_BUCKET = os.environ['CLOUD_STORAGE_BUCKET'][5:]
 
 
 @contextlib.contextmanager
@@ -32,7 +39,20 @@ def capture():
 
 
 class TestRunnerHandler(webapp2.RequestHandler):
+
     def get(self):
+        # Set up test directory.
+        if os.path.isdir('tests'):
+          shutil.rmtree('tests')
+        os.mkdir('tests')
+
+        # Refresh the tests from cloud storage.
+        bucket = storage.Client().get_bucket(GCLOUD_STORAGE_BUCKET)
+        blob_iter = bucket.list_blobs()
+        for blob in blob_iter:
+          blob.download_to_filename('tests/%s' % blob.name)
+
+        # Run the tests.
         with capture() as outf:
             result = pytest.main(['tests'])
 

tests/e2e/app/requirements.txt

@@ -1,3 +1,4 @@
+gcloud
 pytest
 webapp2
 webob

tests/e2e/client/Makefile

@@ -1,7 +1,16 @@
 include ../../../common.mk
 
 .PHONY: all
-all:
+all: upload test
+
+.PHONY: test
+test:
 	virtualenv env
 	env/bin/pip install -r requirements.txt
 	env/bin/python main.py https://${GCLOUD_PROJECT}.appspot.com/
+
+.PHONY: upload
+upload:
+	-e gsutil mb ${BUCKET} &> /dev/null
+	gsutil rm ${BUCKET}/**
+	gsutil cp ../tests/* ${BUCKET} &> /dev/null

tests/e2e/tests/ndb_test.py

@@ -1,4 +1,5 @@
 import os
+import pytest
 
 from google.appengine.ext.ndb import model, tasklets