test: make system test timeout configurable and default to 60 seconds (#217)

b/173067462

It seems 30 seconds are too short for mtls test (which uses the system test, and runs on an internal platform). This makes the mtls test very flaky. This PR introduces a `SPANNER_OPERATION_TIMEOUT_IN_SECONDS` env var to make the timeout configurable. 

The default value is now 60 seconds.

Author: arithmetic1728

tests/system/test_system.py

@@ -55,6 +55,9 @@
 CREATE_INSTANCE = os.getenv("GOOGLE_CLOUD_TESTS_CREATE_SPANNER_INSTANCE") is not None
 USE_EMULATOR = os.getenv("SPANNER_EMULATOR_HOST") is not None
 SKIP_BACKUP_TESTS = os.getenv("SKIP_BACKUP_TESTS") is not None
+SPANNER_OPERATION_TIMEOUT_IN_SECONDS = int(
+    os.getenv("SPANNER_OPERATION_TIMEOUT_IN_SECONDS", 60)
+)
 
 if CREATE_INSTANCE:
     INSTANCE_ID = "google-cloud" + unique_resource_id("-")
@@ -149,7 +152,9 @@ def setUpModule():
             INSTANCE_ID, config_name, labels=labels
         )
         created_op = Config.INSTANCE.create()
-        created_op.result(30)  # block until completion
+        created_op.result(
+            SPANNER_OPERATION_TIMEOUT_IN_SECONDS
+        )  # block until completion
 
     else:
         Config.INSTANCE = Config.CLIENT.instance(INSTANCE_ID)
@@ -208,7 +213,9 @@ def test_create_instance(self):
         self.instances_to_delete.append(instance)
 
         # We want to make sure the operation completes.
-        operation.result(30)  # raises on failure / timeout.
+        operation.result(
+            SPANNER_OPERATION_TIMEOUT_IN_SECONDS
+        )  # raises on failure / timeout.
 
         # Create a new instance instance and make sure it is the same.
         instance_alt = Config.CLIENT.instance(
@@ -227,7 +234,9 @@ def test_update_instance(self):
         operation = Config.INSTANCE.update()
 
         # We want to make sure the operation completes.
-        operation.result(30)  # raises on failure / timeout.
+        operation.result(
+            SPANNER_OPERATION_TIMEOUT_IN_SECONDS
+        )  # raises on failure / timeout.
 
         # Create a new instance instance and reload it.
         instance_alt = Config.CLIENT.instance(INSTANCE_ID, None)
@@ -308,7 +317,9 @@ def setUpClass(cls):
             cls.DATABASE_NAME, ddl_statements=ddl_statements, pool=pool
         )
         operation = cls._db.create()
-        operation.result(30)  # raises on failure / timeout.
+        operation.result(
+            SPANNER_OPERATION_TIMEOUT_IN_SECONDS
+        )  # raises on failure / timeout.
 
     @classmethod
     def tearDownClass(cls):
@@ -337,7 +348,9 @@ def test_create_database(self):
         self.to_delete.append(temp_db)
 
         # We want to make sure the operation completes.
-        operation.result(30)  # raises on failure / timeout.
+        operation.result(
+            SPANNER_OPERATION_TIMEOUT_IN_SECONDS
+        )  # raises on failure / timeout.
 
         database_ids = [database.name for database in Config.INSTANCE.list_databases()]
         self.assertIn(temp_db.name, database_ids)
@@ -483,8 +496,8 @@ def setUpClass(cls):
         cls._dbs = [db1, db2]
         op1 = db1.create()
         op2 = db2.create()
-        op1.result(30)  # raises on failure / timeout.
-        op2.result(30)  # raises on failure / timeout.
+        op1.result(SPANNER_OPERATION_TIMEOUT_IN_SECONDS)  # raises on failure / timeout.
+        op2.result(SPANNER_OPERATION_TIMEOUT_IN_SECONDS)  # raises on failure / timeout.
 
         current_config = Config.INSTANCE.configuration_name
         same_config_instance_id = "same-config" + unique_resource_id("-")
@@ -494,7 +507,7 @@ def setUpClass(cls):
             same_config_instance_id, current_config, labels=labels
         )
         op = cls._same_config_instance.create()
-        op.result(30)
+        op.result(SPANNER_OPERATION_TIMEOUT_IN_SECONDS)
         cls._instances = [cls._same_config_instance]
 
         retry = RetryErrors(exceptions.ServiceUnavailable)
@@ -513,7 +526,7 @@ def setUpClass(cls):
                 diff_config_instance_id, diff_configs[0], labels=labels
             )
             op = cls._diff_config_instance.create()
-            op.result(30)
+            op.result(SPANNER_OPERATION_TIMEOUT_IN_SECONDS)
             cls._instances.append(cls._diff_config_instance)
 
     @classmethod
@@ -675,7 +688,7 @@ def test_multi_create_cancel_update_error_restore_errors(self):
             return
         new_db = self._diff_config_instance.database("diff_config")
         op = new_db.create()
-        op.result(30)
+        op.result(SPANNER_OPERATION_TIMEOUT_IN_SECONDS)
         self.to_drop.append(new_db)
         with self.assertRaises(exceptions.InvalidArgument):
             new_db.restore(source=backup1)
@@ -866,7 +879,9 @@ def setUpClass(cls):
             cls.DATABASE_NAME, ddl_statements=ddl_statements, pool=pool
         )
         operation = cls._db.create()
-        operation.result(30)  # raises on failure / timeout.
+        operation.result(
+            SPANNER_OPERATION_TIMEOUT_IN_SECONDS
+        )  # raises on failure / timeout.
 
     @classmethod
     def tearDownClass(cls):
@@ -1788,7 +1803,9 @@ def test_read_w_index(self):
         self.to_delete.append(_DatabaseDropper(temp_db))
 
         # We want to make sure the operation completes.
-        operation.result(30)  # raises on failure / timeout.
+        operation.result(
+            SPANNER_OPERATION_TIMEOUT_IN_SECONDS
+        )  # raises on failure / timeout.
         committed = self._set_up_table(row_count, database=temp_db)
 
         with temp_db.snapshot(read_timestamp=committed) as snapshot:

tests/system/test_system_dbapi.py

@@ -39,6 +39,11 @@
 )
 
 
+SPANNER_OPERATION_TIMEOUT_IN_SECONDS = int(
+    os.getenv("SPANNER_OPERATION_TIMEOUT_IN_SECONDS", 60)
+)
+
+
 def setUpModule():
     if USE_EMULATOR:
         from google.auth.credentials import AnonymousCredentials
@@ -91,7 +96,9 @@ def setUpModule():
             INSTANCE_ID, config_name, labels=labels
         )
         created_op = Config.INSTANCE.create()
-        created_op.result(30)  # block until completion
+        created_op.result(
+            SPANNER_OPERATION_TIMEOUT_IN_SECONDS
+        )  # block until completion
 
     else:
         Config.INSTANCE = Config.CLIENT.instance(INSTANCE_ID)
@@ -126,7 +133,9 @@ def setUpClass(cls):
             ddl_statements=cls.DDL_STATEMENTS,
             pool=BurstyPool(labels={"testcase": "database_api"}),
         )
-        cls._db.create().result(30)  # raises on failure / timeout.
+        cls._db.create().result(
+            SPANNER_OPERATION_TIMEOUT_IN_SECONDS
+        )  # raises on failure / timeout.
 
     @classmethod
     def tearDownClass(cls):