PYCBC-1697: Update collection/bucket mgmt tests that assume default engine is Couchstore

Motivation
==========
In Morpheus the default storage engine is Magma, and some collection/bucket management tests that use the default bucket when they require a Couchstore bucket, are failing

Changes
=======
* Update history retention tests that verify that FeatureUnavailable is raised for Couchstore, to create a temporary Couchstore bucket.
* Update the 'replica_index' bucket management tests to create a temporary Couchstore bucket. This is a views-related bucket parameter that's not available in Magma.

Results
=======
Tests pass

Change-Id: I00b317433de082987cc78f37429227a0eb7ca234
Reviewed-on: https://review.couchbase.org/c/couchbase-python-client/+/231351
Tested-by: Build Bot <build@couchbase.com>
Reviewed-by: Jared Casey <jared.casey@couchbase.com>

Author: DemetrisChr

acouchbase/tests/bucketmgmt_t.py

@@ -117,6 +117,7 @@ async def test_bucket_create_replica_index_true(self, cb_env, test_bucket):
             CreateBucketSettings(
                 name=test_bucket,
                 bucket_type=BucketType.COUCHBASE,
+                storage_backend=StorageBackend.COUCHSTORE,
                 ram_quota_mb=100,
                 replica_index=True))
         bucket = await cb_env.try_n_times(10, 1, cb_env.bm.get_bucket, test_bucket)
@@ -130,6 +131,7 @@ async def test_bucket_create_replica_index_false(self, cb_env, test_bucket):
             CreateBucketSettings(
                 name=test_bucket,
                 bucket_type=BucketType.COUCHBASE,
+                storage_backend=StorageBackend.COUCHSTORE,
                 ram_quota_mb=100,
                 replica_index=False))
         bucket = await cb_env.try_n_times(10, 1, cb_env.bm.get_bucket, test_bucket)

acouchbase/tests/collectionmgmt_t.py

@@ -21,8 +21,7 @@
 import pytest_asyncio
 
 from acouchbase.cluster import get_event_loop
-from couchbase.exceptions import (BucketDoesNotExistException,
-                                  CollectionAlreadyExistsException,
+from couchbase.exceptions import (CollectionAlreadyExistsException,
                                   CollectionNotFoundException,
                                   DocumentNotFoundException,
                                   FeatureUnavailableException,
@@ -327,13 +326,12 @@ async def test_drop_collection_scope_not_found(self, cb_env):
         with pytest.raises(ScopeNotFoundException):
             await cb_env.test_bucket_cm.drop_collection("fake-scope", "fake-collection")
 
-    @pytest.mark.usefixtures('cleanup_collection')
     @pytest.mark.usefixtures('check_non_deduped_history_supported')
     @pytest.mark.asyncio
     async def test_create_collection_history_retention(self, cb_env):
-        bucket_name = 'test-magma-bucket'
+        bucket_name = f'test-bucket-{str(uuid4())[:6]}'
         scope_name = '_default'
-        collection_name = self.TEST_COLLECTION
+        collection_name = 'test-collection'
 
         await cb_env.create_bucket(bucket_name, storage_backend=StorageBackend.MAGMA)
         bucket = cb_env.cluster.bucket(bucket_name)
@@ -350,36 +348,38 @@ async def test_create_collection_history_retention(self, cb_env):
         assert collection_spec is not None
         assert collection_spec.history
 
-        await cb_env.try_n_times_till_exception(10,
-                                                3,
-                                                cb_env.bm.drop_bucket,
-                                                bucket_name,
-                                                expected_exceptions=(BucketDoesNotExistException,))
+        await cb_env.purge_buckets([bucket_name])
 
-    @pytest.mark.usefixtures('cleanup_collection')
     @pytest.mark.usefixtures('check_non_deduped_history_supported')
     @pytest.mark.asyncio
     async def test_create_collection_history_retention_unsupported(self, cb_env):
+        bucket_name = f'test-bucket-{str(uuid4())[:6]}'
         scope_name = '_default'
-        collection_name = self.TEST_COLLECTION
+        collection_name = 'test-collection'
+
+        await cb_env.create_bucket(bucket_name, storage_backend=StorageBackend.COUCHSTORE)
+        bucket = cb_env.cluster.bucket(bucket_name)
+        await cb_env.try_n_times(10, 1, bucket.on_connect)
+        cm = bucket.collections()
 
         # Couchstore does not support history retention
         with pytest.raises(FeatureUnavailableException):
-            await cb_env.test_bucket_cm.create_collection(
+            await cm.create_collection(
                 scope_name, collection_name, CreateCollectionSettings(history=True))
 
         with pytest.raises(FeatureUnavailableException):
-            await cb_env.test_bucket_cm.create_collection(
+            await cm.create_collection(
                 scope_name, collection_name, CreateCollectionSettings(history=False))
 
-    @pytest.mark.usefixtures('cleanup_collection')
+        await cb_env.purge_buckets([bucket_name])
+
     @pytest.mark.usefixtures('check_non_deduped_history_supported')
     @pytest.mark.usefixtures('check_update_collection_supported')
     @pytest.mark.asyncio
     async def test_update_collection_history_retention(self, cb_env):
-        bucket_name = 'test-magma-bucket'
+        bucket_name = f'test-bucket-{str(uuid4())[:6]}'
         scope_name = '_default'
-        collection_name = self.TEST_COLLECTION
+        collection_name = 'test-collection'
 
         await cb_env.create_bucket(bucket_name, storage_backend=StorageBackend.MAGMA)
         bucket = cb_env.cluster.bucket(bucket_name)
@@ -406,35 +406,39 @@ async def test_update_collection_history_retention(self, cb_env):
         assert collection_spec is not None
         assert collection_spec.history
 
-        await cb_env.try_n_times_till_exception(10,
-                                                3,
-                                                cb_env.bm.drop_bucket,
-                                                bucket_name,
-                                                expected_exceptions=(BucketDoesNotExistException,))
+        await cb_env.purge_buckets([bucket_name])
 
     @pytest.mark.usefixtures("cleanup_collection")
     @pytest.mark.usefixtures('check_non_deduped_history_supported')
     @pytest.mark.usefixtures('check_update_collection_supported')
     @pytest.mark.asyncio
     async def test_update_collection_history_retention_unsupported(self, cb_env):
+        bucket_name = f'test-bucket-{str(uuid4())[:6]}'
         scope_name = '_default'
-        collection_name = self.TEST_COLLECTION
+        collection_name = 'test-collection'
 
-        await cb_env.test_bucket_cm.create_collection(scope_name, collection_name)
-        collection_spec = await cb_env.get_collection(scope_name, collection_name)
+        await cb_env.create_bucket(bucket_name, storage_backend=StorageBackend.COUCHSTORE)
+        bucket = cb_env.cluster.bucket(bucket_name)
+        await cb_env.try_n_times(10, 1, bucket.on_connect)
+        cm = bucket.collections()
+
+        await cm.create_collection(scope_name, collection_name)
+        collection_spec = await cb_env.get_collection(scope_name, collection_name, bucket_name)
         assert collection_spec is not None
         assert collection_spec.history is False
 
         # Couchstore does not support history retention
         with pytest.raises(FeatureUnavailableException):
-            await cb_env.test_bucket_cm.update_collection(
+            await cm.update_collection(
                 scope_name, collection_name, UpdateCollectionSettings(history=True))
 
         # Collection history retention setting remains unchanged
-        collection_spec = await cb_env.get_collection(scope_name, collection_name)
+        collection_spec = await cb_env.get_collection(scope_name, collection_name, bucket_name)
         assert collection_spec is not None
         assert collection_spec.history is False
 
+        await cb_env.purge_buckets([bucket_name])
+
     @pytest.mark.usefixtures('cleanup_collection')
     @pytest.mark.usefixtures('check_update_collection_supported')
     @pytest.mark.usefixtures('check_update_collection_max_expiry_supported')

couchbase/tests/bucketmgmt_t.py

@@ -209,6 +209,7 @@ def test_bucket_create_replica_index_false(self, cb_env):
             CreateBucketSettings(
                 name=bucket_name,
                 bucket_type=BucketType.COUCHBASE,
+                storage_backend=StorageBackend.COUCHSTORE,  # Views only supported on CouchStore
                 ram_quota_mb=100,
                 replica_index=False))
         cb_env.consistency.wait_until_bucket_present(bucket_name)
@@ -223,6 +224,7 @@ def test_bucket_create_replica_index_true(self, cb_env):
             CreateBucketSettings(
                 name=bucket_name,
                 bucket_type=BucketType.COUCHBASE,
+                storage_backend=StorageBackend.COUCHSTORE,  # Views only supported on CouchStore
                 ram_quota_mb=100,
                 replica_index=True))
         cb_env.consistency.wait_until_bucket_present(bucket_name)

couchbase/tests/collectionmgmt_t.py

@@ -17,8 +17,7 @@
 
 import pytest
 
-from couchbase.exceptions import (BucketDoesNotExistException,
-                                  CollectionAlreadyExistsException,
+from couchbase.exceptions import (CollectionAlreadyExistsException,
                                   CollectionNotFoundException,
                                   DocumentNotFoundException,
                                   FeatureUnavailableException,
@@ -95,6 +94,11 @@ def check_negative_collection_max_expiry_supported(self, cb_env):
                                                        cb_env.server_version_short,
                                                        cb_env.mock_server_type)
 
+    @pytest.fixture()
+    def drop_bucket(self, cb_env):
+        yield
+        cb_env.drop_bucket()
+
     def test_collection_goes_in_correct_bucket(self, cb_env):
         collection_name = cb_env.get_collection_name()
         scope_name = '_default'
@@ -287,89 +291,95 @@ def test_get_all_scopes(self, cb_env):
                 assert c.history is None or isinstance(c.history, bool)
 
     @pytest.mark.usefixtures('check_non_deduped_history_supported')
+    @pytest.mark.usefixtures('drop_bucket')
     def test_create_collection_history_retention(self, cb_env):
-        bucket_name = 'test-magma-bucket'
+        bucket_name = cb_env.get_bucket_name()
         scope_name = '_default'
-        collection_name = cb_env.get_collection_name()
+        collection_name = 'test-collection'
 
         cb_env.create_bucket(bucket_name, storage_backend=StorageBackend.MAGMA)
         bucket = TestEnvironment.try_n_times(3, 5, cb_env.cluster.bucket, bucket_name)
         cm = bucket.collections()
 
         cm.create_collection(scope_name, collection_name, CreateCollectionSettings(history=True))
         cb_env.consistency.wait_until_collection_present(bucket.name, scope_name, collection_name)
+
         collection_spec = cb_env.get_collection(scope_name, collection_name, bucket_name=bucket_name)
         assert collection_spec is not None
         assert collection_spec.history
 
-        TestEnvironment.try_n_times_till_exception(10,
-                                                   3,
-                                                   cb_env.bm.drop_bucket,
-                                                   bucket_name,
-                                                   expected_exceptions=(BucketDoesNotExistException,))
-
     @pytest.mark.usefixtures('check_non_deduped_history_supported')
+    @pytest.mark.usefixtures('drop_bucket')
     def test_create_collection_history_retention_unsupported(self, cb_env):
+        bucket_name = cb_env.get_bucket_name()
         scope_name = '_default'
-        collection_name = cb_env.get_collection_name()
+        collection_name = 'test-collection'
+
+        cb_env.create_bucket(bucket_name, storage_backend=StorageBackend.COUCHSTORE)
+        bucket = TestEnvironment.try_n_times(3, 5, cb_env.cluster.bucket, bucket_name)
+        cm = bucket.collections()
 
         # Couchstore does not support history retention
         with pytest.raises(FeatureUnavailableException):
-            cb_env.test_bucket_cm.create_collection(
+            cm.create_collection(
                 scope_name, collection_name, CreateCollectionSettings(history=True))
 
         with pytest.raises(FeatureUnavailableException):
-            cb_env.test_bucket_cm.create_collection(
+            cm.create_collection(
                 scope_name, collection_name, CreateCollectionSettings(history=False))
 
     @pytest.mark.usefixtures('check_non_deduped_history_supported')
     @pytest.mark.usefixtures('check_update_collection_supported')
+    @pytest.mark.usefixtures('drop_bucket')
     def test_update_collection_history_retention(self, cb_env):
-        bucket_name = 'test-magma-bucket'
+        bucket_name = cb_env.get_bucket_name()
         scope_name = '_default'
-        collection_name = cb_env.get_collection_name()
+        collection_name = 'test-collection'
 
         cb_env.create_bucket(bucket_name, storage_backend=StorageBackend.MAGMA)
         bucket = TestEnvironment.try_n_times(3, 5, cb_env.cluster.bucket, bucket_name)
         cm = bucket.collections()
 
         cm.create_collection(scope_name, collection_name, CreateCollectionSettings(history=False))
         cb_env.consistency.wait_until_collection_present(bucket.name, scope_name, collection_name)
+
         collection_spec = cb_env.get_collection(scope_name, collection_name, bucket_name=bucket_name)
         assert collection_spec is not None
         assert not collection_spec.history
 
         cm.update_collection(scope_name, collection_name, UpdateCollectionSettings(history=True))
         cb_env.consistency.wait_until_collection_has_settings(
             bucket.name, scope_name, collection_name, {'history': True})
+
         collection_spec = cb_env.get_collection(scope_name, collection_name, bucket_name=bucket_name)
         assert collection_spec is not None
         assert collection_spec.history
 
-        TestEnvironment.try_n_times_till_exception(10,
-                                                   3,
-                                                   cb_env.bm.drop_bucket,
-                                                   bucket_name,
-                                                   expected_exceptions=(BucketDoesNotExistException,))
-
     @pytest.mark.usefixtures('check_non_deduped_history_supported')
     @pytest.mark.usefixtures('check_update_collection_supported')
+    @pytest.mark.usefixtures('drop_bucket')
     def test_update_collection_history_retention_unsupported(self, cb_env):
+        bucket_name = cb_env.get_bucket_name()
         scope_name = '_default'
-        collection_name = cb_env.get_collection_name()
+        collection_name = 'test-collection'
 
-        cb_env.test_bucket_cm.create_collection(scope_name, collection_name)
-        cb_env.consistency.wait_until_collection_present(cb_env.test_bucket.name, scope_name, collection_name)
-        collection_spec = cb_env.get_collection(scope_name, collection_name)
+        cb_env.create_bucket(bucket_name, storage_backend=StorageBackend.COUCHSTORE)
+        bucket = TestEnvironment.try_n_times(3, 5, cb_env.cluster.bucket, bucket_name)
+        cm = bucket.collections()
+
+        cm.create_collection(scope_name, collection_name)
+        cb_env.consistency.wait_until_collection_present(bucket_name, scope_name, collection_name)
+
+        collection_spec = cb_env.get_collection(scope_name, collection_name, bucket_name)
         assert collection_spec is not None
         assert collection_spec.history is False
 
         # Couchstore does not support history retention
         with pytest.raises(FeatureUnavailableException):
-            cb_env.test_bucket_cm.update_collection(scope_name, collection_name, UpdateCollectionSettings(history=True))
+            cm.update_collection(scope_name, collection_name, UpdateCollectionSettings(history=True))
 
         # Collection history retention setting remains unchanged
-        collection_spec = cb_env.get_collection(scope_name, collection_name)
+        collection_spec = cb_env.get_collection(scope_name, collection_name, bucket_name)
         assert collection_spec is not None
         assert collection_spec.history is False
 

tests/environments/collection_mgmt_environment.py

@@ -52,6 +52,23 @@ def get_scope_names(self):
         available_ids = self._used_scope_ids.difference(self._dropped_scope_ids)
         return list(available_ids)
 
+    def drop_bucket(self):
+        available_ids = self._used_bucket_ids.difference(self._dropped_bucket_ids)
+        bucket_name = available_ids.pop()
+        self._dropped_bucket_ids.add(bucket_name)
+        try:
+            self.bm.drop_bucket(bucket_name)
+        except BucketDoesNotExistException:
+            pass
+        self.consistency.wait_until_bucket_dropped(bucket_name)
+
+    def get_bucket_name(self):
+        all_ids = set(self._bucket_ids)
+        available_ids = all_ids.difference(self._used_bucket_ids)
+        bucket_id = random.choice(list(available_ids))
+        self._used_bucket_ids.add(bucket_id)
+        return bucket_id
+
     def setup(self):
         TestEnvironment.try_n_times(3, 5, self.setup_collection_mgmt, self.TEST_BUCKET)
         self._batch_id = str(uuid.uuid4())[:8]
@@ -61,6 +78,9 @@ def setup(self):
         self._used_collection_ids = set()
         self._dropped_collection_ids = set()
         self._collection_ids = [f'{self._batch_id}_collection_{i}' for i in range(50)]
+        self._bucket_ids = [f'{self._batch_id}_bucket_{i}' for i in range(10)]
+        self._used_bucket_ids = set()
+        self._dropped_bucket_ids = set()
 
     def teardown(self):
         for c in self._used_collection_ids: