From 812885048932ed6b1ef229791662ac0fcd550ac3 Mon Sep 17 00:00:00 2001 From: Fitz Elliott Date: Mon, 25 May 2026 00:57:20 -0400 Subject: [PATCH 1/2] update oci provider docstring --- waterbutler/providers/oraclecloud/provider.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/waterbutler/providers/oraclecloud/provider.py b/waterbutler/providers/oraclecloud/provider.py index 28534a056..663d5b5bf 100644 --- a/waterbutler/providers/oraclecloud/provider.py +++ b/waterbutler/providers/oraclecloud/provider.py @@ -70,24 +70,26 @@ class OracleCloudProvider(BaseProvider): * **Multipart uploads** -- S3 multipart upload API is supported by OCI S3 compat. Would be needed for files > ~5 GB. - * **Presigned URLs** -- S3 query-string SigV4 (presigned URLs) works with OCI. - Could replace the need for OCI Pre-Authenticated Requests (PARs). * **Folder intra-copy** -- the current ``intra_copy`` implementation only handles individual files; recursive folder copy would walk the prefix and - issue one ``PUT Object - Copy`` per key. + issue one ``PUT Object - Copy`` per key. NOT NEEDED FOR LIMITED PROVIDER * **Metadata pagination** -- ``_metadata_folder`` reads only the first page of ListObjectsV2 results; ``_list_keys_under_prefix`` (used by folder - delete) already paginates correctly and can serve as the template. + delete) already paginates correctly and can serve as the template. NOT + NEEDED FOR LIMITED PROVIDER Cannot be done via S3-compatible API (OCI-native only): * **Work Request polling** -- OCI-specific async operation tracking for long-running tasks. S3 compat does not expose this. Multipart upload is the S3 equivalent - for large operations. + for large operations. NOT NEEDED FOR LIMITED PROVIDER * **Storage tier management** -- OCI storage tiers (Standard, InfrequentAccess, - Archive) and archival-state transitions require the native OCI API. - * **Object lifecycle policies** -- native OCI API only. - * **Namespace/compartment management** -- native OCI API only. + Archive) and archival-state transitions require the native OCI API. NOT NEEDED + FOR LIMITED PROVIDER + * **Object lifecycle policies** -- native OCI API only. NOT NEEDED FOR LIMITED + PROVIDER + * **Namespace/compartment management** -- native OCI API only. NOT NEEDED FOR + LIMITED PROVIDER """ NAME = "oraclecloud" From 259dfbfd6622673b12da62428d7721b97e9e5ec1 Mon Sep 17 00:00:00 2001 From: Andriy Sheredko Date: Tue, 9 Jun 2026 17:28:10 +0300 Subject: [PATCH 2/2] =?UTF-8?q?ENG-11274=20=E2=80=94=20test=20cleanups=20&?= =?UTF-8?q?=20fill-out?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tests/providers/oraclecloud/test_provider.py | 347 +++++++++++++------ tests/providers/oraclecloud/test_signing.py | 30 ++ 2 files changed, 272 insertions(+), 105 deletions(-) diff --git a/tests/providers/oraclecloud/test_provider.py b/tests/providers/oraclecloud/test_provider.py index 37ad1b31d..3114bf662 100644 --- a/tests/providers/oraclecloud/test_provider.py +++ b/tests/providers/oraclecloud/test_provider.py @@ -13,10 +13,7 @@ from waterbutler.core.path import WaterButlerPath from waterbutler.core.streams import ResponseStreamReader, StringStream from waterbutler.providers.oraclecloud import OracleCloudProvider -from waterbutler.providers.oraclecloud.metadata import ( - OracleCloudFileMetadata, - OracleCloudFolderMetadata, -) +from waterbutler.providers.oraclecloud.metadata import OracleCloudFileMetadata @pytest.fixture() @@ -57,6 +54,27 @@ def _mock_response(status=200, headers=None, body=b"", text=""): return resp +def _request_sequence(items): + """``make_request`` side effect that drains any streamed body, then yields each + queued response in order (an item may be an exception to raise instead). + """ + queue = list(items) + + async def _side_effect(method, url, *args, **kwargs): + data = kwargs.get("data") + if data is not None and hasattr(data, "read"): + # Drain in positive-size chunks the way aiohttp does, so tee'd hash + # writers see the body exactly once. + while await data.read(8192): + pass + item = queue.pop(0) + if isinstance(item, Exception): + raise item + return item + + return _side_effect + + class TestProviderInit: def test_provider_init(self, mock_provider, mock_settings): @@ -132,6 +150,17 @@ def test_can_intra_move_file(self, mock_provider, file_wb_path): def test_can_intra_move_folder(self, mock_provider, folder_wb_path): assert not mock_provider.can_intra_move(mock_provider, folder_wb_path) + def test_can_intra_copy_other_provider(self, mock_provider, file_wb_path): + assert not mock_provider.can_intra_copy(mock.Mock(), file_wb_path) + + def test_can_intra_copy_different_endpoint( + self, mock_provider, mock_auth, mock_creds, mock_settings, file_wb_path + ): + other = OracleCloudProvider( + mock_auth, mock_creds, {**mock_settings, "namespace": "other-ns"} + ) + assert not mock_provider.can_intra_copy(other, file_wb_path) + class TestMetadata: @@ -161,19 +190,23 @@ async def test_metadata_file(self, mock_provider, file_wb_path): assert metadata.size == 1024 assert metadata.etag == "abc123" - # @pytest.mark.asyncio - # async def test_metadata_file_not_found(self, mock_provider, file_wb_path): + @pytest.mark.asyncio + async def test_metadata_file_not_found(self, mock_provider, file_wb_path): - # head_resp = _mock_response(status=404) + with mock.patch.object( + mock_provider, + "make_request", + new_callable=mock.AsyncMock, + side_effect=exceptions.MetadataError("not found", code=HTTPStatus.NOT_FOUND), + ): + with pytest.raises(exceptions.MetadataError): + await mock_provider.metadata(file_wb_path) + + @pytest.mark.asyncio + async def test_metadata_folder_raises(self, mock_provider, folder_wb_path): - # with mock.patch.object( - # mock_provider, - # "make_request", - # new_callable=mock.AsyncMock, - # return_value=head_resp, - # ): - # with pytest.raises(exceptions.MetadataError): - # await mock_provider.metadata(file_wb_path) + with pytest.raises(exceptions.MetadataError): + await mock_provider.metadata(folder_wb_path) class TestCRUD: @@ -238,108 +271,145 @@ async def test_download_with_range(self, mock_provider, file_wb_path): assert isinstance(stream, ResponseStreamReader) # Verify Range header was included in the signed headers call_kwargs = mocked.call_args - assert "Range" in call_kwargs.kwargs["headers"] - - # @pytest.mark.asyncio - # async def test_upload_file(self, mock_provider, file_wb_path, file_content): - - # expected_md5 = hashlib.md5(file_content).hexdigest() - - # # exists check (HEAD) returns 404 -> file is new - # head_404 = _mock_response(status=404) - # # PUT returns 200 with ETag - # put_resp = _mock_response( - # status=200, headers={"ETag": f'"{expected_md5}"'} - # ) - # # post-upload metadata fetch (HEAD) returns 200 - # head_200 = _mock_response( - # status=200, - # headers={ - # "Content-Type": "application/octet-stream", - # "Content-Length": str(len(file_content)), - # "ETag": f'"{expected_md5}"', - # "Last-Modified": "Thu, 01 Mar 2025 19:04:45 GMT", - # }, - # ) - - # with mock.patch.object( - # mock_provider, - # "make_request", - # new_callable=mock.AsyncMock, - # side_effect=[head_404, put_resp, head_200], - # ): - # metadata, created = await mock_provider.upload( - # StringStream(file_content), file_wb_path - # ) - - # assert created is True - # assert isinstance(metadata, OracleCloudFileMetadata) - # assert metadata.name == "text-file-1.txt" - - # @pytest.mark.asyncio - # async def test_upload_existing_file(self, mock_provider, file_wb_path, file_content): - - # expected_md5 = hashlib.md5(file_content).hexdigest() - - # # exists check (HEAD) returns 200 -> file already exists - # head_exists = _mock_response( - # status=200, - # headers={ - # "Content-Type": "text/plain", - # "Content-Length": "100", - # "ETag": '"oldmd5"', - # "Last-Modified": "Thu, 01 Mar 2025 00:00:00 GMT", - # }, - # ) - # # PUT returns 200 - # put_resp = _mock_response( - # status=200, headers={"ETag": f'"{expected_md5}"'} - # ) - # # post-upload metadata fetch - # head_200 = _mock_response( - # status=200, - # headers={ - # "Content-Type": "application/octet-stream", - # "Content-Length": str(len(file_content)), - # "ETag": f'"{expected_md5}"', - # "Last-Modified": "Thu, 01 Mar 2025 19:04:45 GMT", - # }, - # ) - - # with mock.patch.object( - # mock_provider, - # "make_request", - # new_callable=mock.AsyncMock, - # side_effect=[head_exists, put_resp, head_200], - # ): - # metadata, created = await mock_provider.upload( - # StringStream(file_content), file_wb_path - # ) - - # assert created is False - # assert isinstance(metadata, OracleCloudFileMetadata) + assert call_kwargs.kwargs["headers"]["Range"] == "bytes=0-6" @pytest.mark.asyncio - async def test_upload_checksum_mismatch( - self, mock_provider, file_wb_path, file_content - ): + async def test_download_open_ended_range(self, mock_provider, file_wb_path): - # exists check returns 404 - head_404 = _mock_response(status=404) - # PUT returns 200 with wrong ETag - put_resp = _mock_response( - status=200, headers={"ETag": '"mismatch"'} + get_resp = _mock_response( + status=206, body=b"tail", headers={"Content-Length": "4"} ) with mock.patch.object( mock_provider, "make_request", new_callable=mock.AsyncMock, - side_effect=[head_404, put_resp], - ): + return_value=get_resp, + ) as mocked: + await mock_provider.download(file_wb_path, range=(5, None)) + + assert mocked.call_args.kwargs["headers"]["Range"] == "bytes=5-" + + @pytest.mark.asyncio + async def test_upload_file(self, mock_provider, file_wb_path, file_content): + + expected_md5 = hashlib.md5(file_content).hexdigest() + put_resp = _mock_response(status=200, headers={"ETag": f'"{expected_md5}"'}) + head_200 = _mock_response( + status=200, + headers={ + "Content-Type": "application/octet-stream", + "Content-Length": str(len(file_content)), + "ETag": f'"{expected_md5}"', + "Last-Modified": "Thu, 01 Mar 2025 19:04:45 GMT", + }, + ) + # exists() HEAD raises 404, PUT succeeds, post-upload HEAD returns metadata + sequence = _request_sequence([ + exceptions.MetadataError("not found", code=HTTPStatus.NOT_FOUND), + put_resp, + head_200, + ]) + + with mock.patch.object(mock_provider, "make_request", side_effect=sequence): + metadata, created = await mock_provider.upload( + StringStream(file_content), file_wb_path + ) + + assert created is True + assert isinstance(metadata, OracleCloudFileMetadata) + assert metadata.name == "text-file-1.txt" + assert metadata.etag == expected_md5 + + @pytest.mark.asyncio + async def test_upload_existing_file(self, mock_provider, file_wb_path, file_content): + + expected_md5 = hashlib.md5(file_content).hexdigest() + head_exists = _mock_response( + status=200, + headers={ + "Content-Type": "text/plain", + "Content-Length": "100", + "ETag": '"oldmd5"', + "Last-Modified": "Thu, 01 Mar 2025 00:00:00 GMT", + }, + ) + put_resp = _mock_response(status=200, headers={"ETag": f'"{expected_md5}"'}) + head_200 = _mock_response( + status=200, + headers={ + "Content-Type": "application/octet-stream", + "Content-Length": str(len(file_content)), + "ETag": f'"{expected_md5}"', + "Last-Modified": "Thu, 01 Mar 2025 19:04:45 GMT", + }, + ) + # exists() HEAD finds the object, so created is False + sequence = _request_sequence([head_exists, put_resp, head_200]) + + with mock.patch.object(mock_provider, "make_request", side_effect=sequence): + metadata, created = await mock_provider.upload( + StringStream(file_content), file_wb_path + ) + + assert created is False + assert isinstance(metadata, OracleCloudFileMetadata) + + @pytest.mark.asyncio + async def test_upload_no_etag_skips_verification( + self, mock_provider, file_wb_path, file_content + ): + # No ETag on the PUT response -> integrity check is skipped, upload still succeeds + put_resp = _mock_response(status=200, headers={}) + head_200 = _mock_response( + status=200, + headers={ + "Content-Type": "application/octet-stream", + "Content-Length": str(len(file_content)), + "ETag": '"abc123"', + }, + ) + sequence = _request_sequence([ + exceptions.MetadataError("not found", code=HTTPStatus.NOT_FOUND), + put_resp, + head_200, + ]) + + with mock.patch.object(mock_provider, "make_request", side_effect=sequence): + metadata, created = await mock_provider.upload( + StringStream(file_content), file_wb_path + ) + + assert created is True + assert isinstance(metadata, OracleCloudFileMetadata) + + @pytest.mark.asyncio + async def test_upload_checksum_mismatch( + self, mock_provider, file_wb_path, file_content + ): + + # PUT returns 200 with an ETag that won't match the uploaded body + put_resp = _mock_response(status=200, headers={"ETag": '"deadbeef"'}) + sequence = _request_sequence([ + exceptions.MetadataError("not found", code=HTTPStatus.NOT_FOUND), + put_resp, + ]) + + with mock.patch.object(mock_provider, "make_request", side_effect=sequence): with pytest.raises(exceptions.UploadChecksumMismatchError): await mock_provider.upload(StringStream(file_content), file_wb_path) + @pytest.mark.asyncio + async def test_download_accept_url(self, mock_provider, file_wb_path): + + signed_url = await mock_provider.download(file_wb_path, accept_url=True) + + assert isinstance(signed_url, str) + assert "X-Amz-Algorithm=AWS4-HMAC-SHA256" in signed_url + assert "X-Amz-Credential=fake-access-key-id" in signed_url + assert "X-Amz-Signature=" in signed_url + assert "response-content-disposition" in signed_url + @pytest.mark.asyncio async def test_delete_file(self, mock_provider, file_wb_path): @@ -353,6 +423,73 @@ async def test_delete_file(self, mock_provider, file_wb_path): ): await mock_provider.delete(file_wb_path) + @pytest.mark.asyncio + async def test_delete_folder_raises(self, mock_provider, folder_wb_path): + + with pytest.raises(exceptions.DeleteError): + await mock_provider.delete(folder_wb_path) + + @pytest.mark.asyncio + async def test_delete_not_found(self, mock_provider, file_wb_path): + + with mock.patch.object( + mock_provider, + "make_request", + new_callable=mock.AsyncMock, + side_effect=exceptions.DeleteError("not found", code=HTTPStatus.NOT_FOUND), + ): + with pytest.raises(exceptions.DeleteError): + await mock_provider.delete(file_wb_path) + + +class TestIntraCopy: + + @pytest.mark.asyncio + async def test_intra_copy_file(self, mock_provider, file_wb_path): + + dest_path = WaterButlerPath("/folder-2/copy.txt") + copy_resp = _mock_response(status=200, body=b"") + head_200 = _mock_response( + status=200, + headers={ + "Content-Type": "application/octet-stream", + "Content-Length": "44", + "ETag": '"abc123"', + }, + ) + # exists(dest) HEAD raises 404, copy PUT succeeds, then HEAD for metadata + sequence = _request_sequence([ + exceptions.MetadataError("not found", code=HTTPStatus.NOT_FOUND), + copy_resp, + head_200, + ]) + + with mock.patch.object(mock_provider, "make_request", side_effect=sequence): + metadata, created = await mock_provider.intra_copy( + mock_provider, file_wb_path, dest_path + ) + + assert created is True + assert isinstance(metadata, OracleCloudFileMetadata) + assert metadata.path == "/folder-2/copy.txt" + + @pytest.mark.asyncio + async def test_intra_copy_folder_raises(self, mock_provider, folder_wb_path): + + with pytest.raises(exceptions.CopyError): + await mock_provider.intra_copy( + mock_provider, folder_wb_path, folder_wb_path + ) + + @pytest.mark.asyncio + async def test_intra_copy_file_folder_mismatch_raises( + self, mock_provider, file_wb_path, folder_wb_path + ): + with pytest.raises(exceptions.CopyError): + await mock_provider.intra_copy( + mock_provider, file_wb_path, folder_wb_path + ) + class TestURLBuilding: diff --git a/tests/providers/oraclecloud/test_signing.py b/tests/providers/oraclecloud/test_signing.py index 4f5682087..bcc4b123e 100644 --- a/tests/providers/oraclecloud/test_signing.py +++ b/tests/providers/oraclecloud/test_signing.py @@ -160,3 +160,33 @@ def test_different_payload_hashes_different_signatures(self, frozen_signer): h1 = frozen_signer.sign_request("PUT", url, payload_hash=EMPTY_SHA256) h2 = frozen_signer.sign_request("PUT", url, payload_hash="abc123") assert h1["Authorization"] != h2["Authorization"] + + +class TestSignRequestQuery: + + def test_returns_presigned_url(self, frozen_signer): + url = frozen_signer.sign_request_query( + "GET", + "https://host/bkt/key?response-content-disposition=attachment", + ) + assert isinstance(url, str) + assert "X-Amz-Algorithm=AWS4-HMAC-SHA256" in url + assert "X-Amz-Credential=AKEXAMPLE" in url + assert "X-Amz-Date=20250301T120000Z" in url + assert "X-Amz-Expires=3600" in url + assert "X-Amz-SignedHeaders=host" in url + assert "X-Amz-Signature=" in url + + def test_deterministic(self, frozen_signer): + url = "https://host/bkt/key?response-content-disposition=attachment" + assert ( + frozen_signer.sign_request_query("GET", url) + == frozen_signer.sign_request_query("GET", url) + ) + + def test_preserves_existing_query(self, frozen_signer): + url = frozen_signer.sign_request_query( + "GET", + "https://host/bkt/key?response-content-disposition=attachment", + ) + assert "response-content-disposition=attachment" in url