diff --git a/.librarian/generator-input/.repo-metadata.json b/.librarian/generator-input/.repo-metadata.json index bd870f959..59ebe7f61 100644 --- a/.librarian/generator-input/.repo-metadata.json +++ b/.librarian/generator-input/.repo-metadata.json @@ -12,7 +12,7 @@ "api_id": "storage.googleapis.com", "requires_billing": true, "default_version": "v2", - "codeowner_team": "@googleapis/yoshi-python @googleapis/gcs-sdk-team @googleapis/gcs-fs", + "codeowner_team": "@googleapis/cloud-sdk-python-team @googleapis/gcs-team @googleapis/gcs-fs", "api_shortname": "storage", "api_description": "is a durable and highly available object storage service. Google Cloud Storage is almost infinitely scalable and guarantees consistency: when a write succeeds, the latest copy of the object will be returned to any GET, globally." } diff --git a/.librarian/generator-input/librarian.py b/.librarian/generator-input/librarian.py index ad0baa2a3..ce8a4b92f 100644 --- a/.librarian/generator-input/librarian.py +++ b/.librarian/generator-input/librarian.py @@ -41,6 +41,12 @@ "README.rst", # Exclude autogenerated default import `google.cloud.storage` "google/cloud/storage/*", + # Temporarily exclude google/cloud/_storage_v2/services/storage/client.py + # and tests/unit/gapic due to customization. + # This exclusion will be removed in the google-cloud-python migration PR + # When librarian.py is dropped. + "tests/unit/gapic/*", + "google/cloud/storage_v2/services/storage/client.py", # Exclude autogenerated constraints files for Python 3.7/3.9 "testing/constraints-3.7.txt", "testing/constraints-3.9.txt", diff --git a/.librarian/generator-input/noxfile.py b/.librarian/generator-input/noxfile.py index c9ada0739..a00b72040 100644 --- a/.librarian/generator-input/noxfile.py +++ b/.librarian/generator-input/noxfile.py @@ -17,7 +17,6 @@ from __future__ import absolute_import import os import pathlib -import re import shutil import nox @@ -28,7 +27,13 @@ DEFAULT_PYTHON_VERSION = "3.14" SYSTEM_TEST_PYTHON_VERSIONS = ["3.10", "3.14"] -UNIT_TEST_PYTHON_VERSIONS = ["3.10", "3.11", "3.12", "3.13", "3.14"] +UNIT_TEST_PYTHON_VERSIONS = [ + "3.10", + "3.11", + "3.12", + "3.13", + "3.14", +] CONFORMANCE_TEST_PYTHON_VERSIONS = ["3.12"] CURRENT_DIRECTORY = pathlib.Path(__file__).parent.absolute() @@ -39,6 +44,7 @@ nox.options.sessions = [ "blacken", "conftest_retry", + "conftest_retry_bidi", "docfx", "docs", "lint", @@ -61,9 +67,7 @@ def lint(session): Returns a failure if the linters find linting errors or sufficiently serious code quality issues. """ - # Pin flake8 to 6.0.0 - # See https://github.com/googleapis/python-storage/issues/1102 - session.install("flake8==6.0.0", BLACK_VERSION) + session.install("flake8", BLACK_VERSION) session.run( "black", "--check", @@ -116,6 +120,8 @@ def default(session, install_extras=True): session.install("-e", ".", "-c", constraints_path) + session.run("python", "-m", "pip", "freeze") + # This dependency is included in setup.py for backwards compatibility only # and the client library is expected to pass all tests without it. See # setup.py and README for details. @@ -180,7 +186,14 @@ def system(session): # 2021-05-06: defer installing 'google-cloud-*' to after this package, # in order to work around Python 2.7 googolapis-common-protos # issue. - session.install("mock", "pytest", "pytest-rerunfailures", "-c", constraints_path) + session.install( + "mock", + "pytest", + "pytest-rerunfailures", + "pytest-asyncio", + "-c", + constraints_path, + ) session.install("-e", ".", "-c", constraints_path) session.install( "google-cloud-testutils", @@ -207,30 +220,75 @@ def system(session): @nox.session(python=CONFORMANCE_TEST_PYTHON_VERSIONS) def conftest_retry(session): """Run the retry conformance test suite.""" - conformance_test_folder_path = os.path.join("tests", "conformance") - conformance_test_folder_exists = os.path.exists(conformance_test_folder_path) + json_conformance_tests = "tests/conformance/test_conformance.py" # Environment check: only run tests if found. - if not conformance_test_folder_exists: + if not os.path.exists(json_conformance_tests): session.skip("Conformance tests were not found") + constraints_path = str( + CURRENT_DIRECTORY / "testing" / f"constraints-{session.python}.txt" + ) + # Install all test dependencies and pytest plugin to run tests in parallel. # Then install this package in-place. - session.install("pytest", "pytest-xdist") - session.install("-e", ".") + session.install( + "pytest", + "pytest-xdist", + "-c", + constraints_path, + ) + session.install("-e", ".", "-c", constraints_path) # Run #CPU processes in parallel if no test session arguments are passed in. if session.posargs: test_cmd = [ - "py.test", - "--quiet", - conformance_test_folder_path, + "pytest", + "-vv", + "-s", + json_conformance_tests, *session.posargs, ] else: - test_cmd = ["py.test", "-n", "auto", "--quiet", conformance_test_folder_path] + test_cmd = ["pytest", "-vv", "-s", "-n", "auto", json_conformance_tests] + + # Run pytest against the conformance tests. + session.run(*test_cmd, env={"DOCKER_API_VERSION": "1.39"}) + - # Run py.test against the conformance tests. - session.run(*test_cmd) +@nox.session(python=CONFORMANCE_TEST_PYTHON_VERSIONS) +def conftest_retry_bidi(session): + """Run the retry conformance test suite.""" + + constraints_path = str( + CURRENT_DIRECTORY / "testing" / f"constraints-{session.python}.txt" + ) + + # Install all test dependencies and pytest plugin to run tests in parallel. + # Then install this package in-place. + session.install( + "pytest", + "pytest-xdist", + "pytest-asyncio", + "grpcio", + "grpcio-status", + "grpc-google-iam-v1", + "-c", + constraints_path, + ) + session.install("-e", ".", "-c", constraints_path) + + bidi_tests = [ + "tests/conformance/test_bidi_reads.py", + "tests/conformance/test_bidi_writes.py", + ] + for test_file in bidi_tests: + session.run( + "pytest", + "-vv", + "-s", + test_file, + env={"DOCKER_API_VERSION": "1.39"}, + ) @nox.session(python=DEFAULT_PYTHON_VERSION) diff --git a/.librarian/generator-input/setup.py b/.librarian/generator-input/setup.py index 294e63892..69fc2899e 100644 --- a/.librarian/generator-input/setup.py +++ b/.librarian/generator-input/setup.py @@ -43,10 +43,46 @@ "google-crc32c >= 1.1.3, < 2.0.0", ] extras = { + # TODO: Make these extra dependencies as mandatory once gRPC out of + # experimental in this SDK. More info in b/465352227 + "grpc": [ + "google-api-core[grpc] >= 2.27.0, < 3.0.0", + "grpcio >= 1.33.2, < 2.0.0; python_version < '3.14'", + "grpcio >= 1.75.1, < 2.0.0; python_version >= '3.14'", + "grpcio-status >= 1.76.0, < 2.0.0", + "proto-plus >= 1.22.3, <2.0.0; python_version < '3.13'", + "proto-plus >= 1.25.0, <2.0.0; python_version >= '3.13'", + "protobuf>=3.20.2,<7.0.0,!=4.21.0,!=4.21.1,!=4.21.2,!=4.21.3,!=4.21.4,!=4.21.5", + "grpc-google-iam-v1 >= 0.14.0, <1.0.0", + ], "protobuf": ["protobuf >= 3.20.2, < 7.0.0"], "tracing": [ "opentelemetry-api >= 1.1.0, < 2.0.0", ], + "testing": [ + "google-cloud-testutils", + "numpy", + "psutil", + "py-cpuinfo", + "pytest-benchmark", + "PyYAML", + "mock", + "pytest", + "pytest-cov", + "pytest-asyncio", + "pytest-rerunfailures", + "pytest-xdist", + "google-cloud-testutils", + "google-cloud-iam", + "google-cloud-pubsub", + "google-cloud-kms", + "brotli", + "coverage", + "pyopenssl", + "opentelemetry-sdk", + "flake8", + "black", + ], } @@ -99,7 +135,7 @@ packages=packages, install_requires=dependencies, extras_require=extras, - python_requires=">=3.7", + python_requires=">=3.10", include_package_data=True, zip_safe=False, ) diff --git a/.librarian/state.yaml b/.librarian/state.yaml index 8c3daafaa..7b2efe9ff 100644 --- a/.librarian/state.yaml +++ b/.librarian/state.yaml @@ -2,7 +2,7 @@ image: us-central1-docker.pkg.dev/cloud-sdk-librarian-prod/images-prod/python-li libraries: - id: google-cloud-storage version: 3.10.1 - last_generated_commit: 5400ccce473c439885bd6bf2924fd242271bfcab + last_generated_commit: b3f7a1f7550d68ec7d92d12eb21092930ce16ede apis: - path: google/storage/v2 service_config: storage_v2.yaml @@ -22,10 +22,18 @@ libraries: - ^renovate.json - ^setup.py - ^docs/summary_overview.md - - ^google/cloud/_storage_v2 + - ^google/cloud/_storage_v2/__init__.py + - ^google/cloud/_storage_v2/gapic + - ^google/cloud/_storage_v2/py.typed + - ^google/cloud/_storage_v2/samples/generated_samples + - ^google/cloud/_storage_v2/services/__init__.py + - ^google/cloud/_storage_v2/services/storage/transports + - ^google/cloud/_storage_v2/services/storage/__init__.py + - ^google/cloud/_storage_v2/services/storage/async_client.py + - ^google/cloud/_storage_v2/services/storage/pagers.py + - ^google/cloud/_storage_v2/types - ^samples/generated_samples - ^testing/constraints-3.8.txt - ^testing/constraints-3.1.* - ^tests/__init__.py - - ^tests/unit/gapic tag_format: v{version} diff --git a/google/cloud/_storage_v2/services/storage/async_client.py b/google/cloud/_storage_v2/services/storage/async_client.py index c71ee0472..cd376a088 100644 --- a/google/cloud/_storage_v2/services/storage/async_client.py +++ b/google/cloud/_storage_v2/services/storage/async_client.py @@ -2472,8 +2472,9 @@ def bidi_read_object( across several messages. If an error occurs with any request, the stream closes with a relevant error code. Since you can have multiple outstanding requests, the error response includes a - ``BidiReadObjectRangesError`` field detailing the specific error - for each pending ``read_id``. + ``BidiReadObjectError`` proto in its ``details`` field, + reporting the specific error, if any, for each pending + ``read_id``. **IAM Permissions**: diff --git a/google/cloud/_storage_v2/services/storage/transports/grpc.py b/google/cloud/_storage_v2/services/storage/transports/grpc.py index fae2d7949..6497959d7 100644 --- a/google/cloud/_storage_v2/services/storage/transports/grpc.py +++ b/google/cloud/_storage_v2/services/storage/transports/grpc.py @@ -946,8 +946,9 @@ def bidi_read_object( across several messages. If an error occurs with any request, the stream closes with a relevant error code. Since you can have multiple outstanding requests, the error response includes a - ``BidiReadObjectRangesError`` field detailing the specific error - for each pending ``read_id``. + ``BidiReadObjectError`` proto in its ``details`` field, + reporting the specific error, if any, for each pending + ``read_id``. **IAM Permissions**: diff --git a/google/cloud/_storage_v2/services/storage/transports/grpc_asyncio.py b/google/cloud/_storage_v2/services/storage/transports/grpc_asyncio.py index be54eb3b0..1c7743a72 100644 --- a/google/cloud/_storage_v2/services/storage/transports/grpc_asyncio.py +++ b/google/cloud/_storage_v2/services/storage/transports/grpc_asyncio.py @@ -971,8 +971,9 @@ def bidi_read_object( across several messages. If an error occurs with any request, the stream closes with a relevant error code. Since you can have multiple outstanding requests, the error response includes a - ``BidiReadObjectRangesError`` field detailing the specific error - for each pending ``read_id``. + ``BidiReadObjectError`` proto in its ``details`` field, + reporting the specific error, if any, for each pending + ``read_id``. **IAM Permissions**: diff --git a/google/cloud/_storage_v2/types/storage.py b/google/cloud/_storage_v2/types/storage.py index 8602610be..d1a7c5a36 100644 --- a/google/cloud/_storage_v2/types/storage.py +++ b/google/cloud/_storage_v2/types/storage.py @@ -489,6 +489,11 @@ class ComposeObjectRequest(proto.Message): Optional. The checksums of the complete object. This is validated against the combined checksums of the component objects. + delete_source_objects (bool): + Whether the source objects should be deleted + in the compose request. + + This field is a member of `oneof`_ ``_delete_source_objects``. """ class SourceObject(proto.Message): @@ -580,6 +585,11 @@ class ObjectPreconditions(proto.Message): number=10, message="ObjectChecksums", ) + delete_source_objects: bool = proto.Field( + proto.BOOL, + number=11, + optional=True, + ) class DeleteObjectRequest(proto.Message): @@ -1460,18 +1470,18 @@ class ReadRange(proto.Message): ``ReadObjectRequest`` with ``read_offset`` = -5 and ``read_length`` = 3 would return bytes 10 through 12 of the object. Requesting a negative offset with magnitude larger - than the size of the object returns the entire object. A - ``read_offset`` larger than the size of the object results - in an ``OutOfRange`` error. + than the size of the object is equivalent to ``read_offset`` + = 0. A ``read_offset`` larger than the size of the object + results in an ``OutOfRange`` error. read_length (int): Optional. The maximum number of data bytes the server is allowed to return across all response messages with the same ``read_id``. A ``read_length`` of zero indicates to read until the resource end, and a negative ``read_length`` - causes an error. If the stream returns fewer bytes than - allowed by the ``read_length`` and no error occurred, the - stream includes all data from the ``read_offset`` to the - resource end. + causes an ``OutOfRange`` error. If the stream returns fewer + bytes than allowed by the ``read_length`` and no error + occurred, the stream includes all data from the + ``read_offset`` to the resource end. read_id (int): Required. Read identifier provided by the client. When the client issues more than one outstanding ``ReadRange`` on the @@ -4364,7 +4374,10 @@ class ObjectContexts(proto.Message): Attributes: custom (MutableMapping[str, google.cloud._storage_v2.types.ObjectCustomContextPayload]): - Optional. User-defined object contexts. + Optional. User-defined object contexts. The maximum key or + value size is ``256`` characters. The maximum number of + entries is ``50``. The maximum total serialized size of all + entries is ``25KiB``. """ custom: MutableMapping[str, "ObjectCustomContextPayload"] = proto.MapField( diff --git a/noxfile.py b/noxfile.py index 77823d28d..a00b72040 100644 --- a/noxfile.py +++ b/noxfile.py @@ -67,8 +67,6 @@ def lint(session): Returns a failure if the linters find linting errors or sufficiently serious code quality issues. """ - # Pin flake8 to 6.0.0 - # See https://github.com/googleapis/python-storage/issues/1102 session.install("flake8", BLACK_VERSION) session.run( "black", diff --git a/testing/constraints-3.14.txt b/testing/constraints-3.14.txt index 62739fc5d..2ae5a677e 100644 --- a/testing/constraints-3.14.txt +++ b/testing/constraints-3.14.txt @@ -7,7 +7,7 @@ # Then this file should have google-cloud-foo>=1 google-api-core>=2 google-auth>=2 -grpcio>=1.75.1 +grpcio>=1 proto-plus>=1 protobuf>=6 grpc-google-iam-v1>=0