From 553f62aa9155fc34dfd9f4b0b0adae51f57dfbf2 Mon Sep 17 00:00:00 2001 From: jencymaryjoseph <35571282+jencymaryjoseph@users.noreply.github.com> Date: Wed, 2 Jul 2025 15:09:30 -0700 Subject: [PATCH 01/30] Adding pre-signed URL operation core API and Get Object Request Pojo (#6216) * Add presigned URL request models and tests * Pre-signed URL API and Get Object Request definitions * JavaDoc changes, String to URL change * Delete services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/presignedurl/DefaultPresignedUrlManager.java * Delete services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/presignedurl/PresignedUrlGetObjectRequestMarshaller.java * Delete services/s3/src/test/java/software/amazon/awssdk/services/s3/presignedurl/model/PresignedUrlTest.java * Format PresignedUrlGetObjectRequestWrapper.java * Format PresignedUrlManager.java * Format PresignedUrlGetObjectRequest.java * Format PresignedUrlGetObjectRequestWrapperTest.java * Format PresignedUrlGetObjectRequestTest.java * sonarcube issues fixed * Delete .idea/codeStyles/Project.xml * added consumer builder, javadoc for builder, assert range value test * removing codestle from gitignore --- .../PresignedUrlGetObjectRequestWrapper.java | 153 ++++++++++++++++ .../s3/presignedurl/PresignedUrlManager.java | 84 +++++++++ .../model/PresignedUrlGetObjectRequest.java | 167 ++++++++++++++++++ ...esignedUrlGetObjectRequestWrapperTest.java | 94 ++++++++++ .../PresignedUrlGetObjectRequestTest.java | 115 ++++++++++++ 5 files changed, 613 insertions(+) create mode 100644 services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/presignedurl/model/PresignedUrlGetObjectRequestWrapper.java create mode 100644 services/s3/src/main/java/software/amazon/awssdk/services/s3/presignedurl/PresignedUrlManager.java create mode 100644 services/s3/src/main/java/software/amazon/awssdk/services/s3/presignedurl/model/PresignedUrlGetObjectRequest.java create mode 100644 services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/presignedurl/model/PresignedUrlGetObjectRequestWrapperTest.java create mode 100644 services/s3/src/test/java/software/amazon/awssdk/services/s3/presignedurl/model/PresignedUrlGetObjectRequestTest.java diff --git a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/presignedurl/model/PresignedUrlGetObjectRequestWrapper.java b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/presignedurl/model/PresignedUrlGetObjectRequestWrapper.java new file mode 100644 index 000000000000..efd5a9f78167 --- /dev/null +++ b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/presignedurl/model/PresignedUrlGetObjectRequestWrapper.java @@ -0,0 +1,153 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.services.s3.internal.presignedurl.model; + +import java.net.URL; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.function.Function; +import software.amazon.awssdk.annotations.SdkInternalApi; +import software.amazon.awssdk.core.SdkField; +import software.amazon.awssdk.core.protocol.MarshallLocation; +import software.amazon.awssdk.core.protocol.MarshallingType; +import software.amazon.awssdk.core.traits.LocationTrait; +import software.amazon.awssdk.services.s3.model.S3Request; + +/** + * Internal request object for presigned URL GetObject operations. + *

+ * This class is used internally by the AWS SDK to process presigned URL requests for S3 GetObject operations. It contains minimal + * SdkField definitions needed for custom marshalling and is not intended for direct use by SDK users. + *

+ * Note: This is an internal implementation class and should not be used + * directly. Use {@code PresignedUrlGetObjectRequest} for public API interactions. + */ +@SdkInternalApi +public final class PresignedUrlGetObjectRequestWrapper extends S3Request { + private static final SdkField RANGE_FIELD = SdkField + .builder(MarshallingType.STRING) + .memberName("Range") + .getter(getter(PresignedUrlGetObjectRequestWrapper::range)) + .traits(LocationTrait.builder().location(MarshallLocation.HEADER).locationName("Range") + .unmarshallLocationName("Range").build()).build(); + + private static final List> SDK_FIELDS = Collections.unmodifiableList( + Arrays.asList(RANGE_FIELD)); + + private static final Map> SDK_NAME_TO_FIELD = memberNameToFieldInitializer(); + + private final URL url; + private final String range; + + private PresignedUrlGetObjectRequestWrapper(Builder builder) { + super(builder); + this.url = builder.url; + this.range = builder.range; + } + + public URL url() { + return url; + } + + public String range() { + return range; + } + + @Override + public List> sdkFields() { + return SDK_FIELDS; + } + + @Override + public Map> sdkFieldNameToField() { + return SDK_NAME_TO_FIELD; + } + + private static Function getter(Function g) { + return obj -> g.apply((PresignedUrlGetObjectRequestWrapper) obj); + } + + private static Map> memberNameToFieldInitializer() { + Map> map = new HashMap<>(); + map.put("Range", RANGE_FIELD); + return Collections.unmodifiableMap(map); + } + + @Override + public Builder toBuilder() { + return new Builder(this); + } + + public static Builder builder() { + return new Builder(); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null || getClass() != obj.getClass()) { + return false; + } + if (!super.equals(obj)) { + return false; + } + PresignedUrlGetObjectRequestWrapper that = (PresignedUrlGetObjectRequestWrapper) obj; + return Objects.equals(url, that.url) && Objects.equals(range, that.range); + } + + @Override + public int hashCode() { + int result = Objects.hashCode(super.hashCode()); + result = 31 * result + Objects.hashCode(url); + result = 31 * result + Objects.hashCode(range); + return result; + } + + public static final class Builder extends S3Request.BuilderImpl { + private URL url; + private String range; + + public Builder() { + } + + Builder(PresignedUrlGetObjectRequestWrapper request) { + super(request); + this.url = request.url(); + this.range = request.range(); + } + + public Builder url(URL url) { + this.url = url; + return this; + } + + public Builder range(String range) { + this.range = range; + return this; + } + + @Override + public PresignedUrlGetObjectRequestWrapper build() { + return new PresignedUrlGetObjectRequestWrapper(this); + } + } +} diff --git a/services/s3/src/main/java/software/amazon/awssdk/services/s3/presignedurl/PresignedUrlManager.java b/services/s3/src/main/java/software/amazon/awssdk/services/s3/presignedurl/PresignedUrlManager.java new file mode 100644 index 000000000000..3b005ce871da --- /dev/null +++ b/services/s3/src/main/java/software/amazon/awssdk/services/s3/presignedurl/PresignedUrlManager.java @@ -0,0 +1,84 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.services.s3.presignedurl; + +import software.amazon.awssdk.annotations.SdkPublicApi; +import software.amazon.awssdk.core.exception.SdkClientException; +import software.amazon.awssdk.core.sync.ResponseTransformer; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.model.GetObjectResponse; +import software.amazon.awssdk.services.s3.model.InvalidObjectStateException; +import software.amazon.awssdk.services.s3.model.NoSuchKeyException; +import software.amazon.awssdk.services.s3.model.S3Exception; +import software.amazon.awssdk.services.s3.presignedurl.model.PresignedUrlGetObjectRequest; + +/** + * Interface for executing S3 operations using presigned URLs. This can be accessed using {@link S3Client#presignedUrlManager()}. + */ +@SdkPublicApi +public interface PresignedUrlManager { + /** + *

+ * Downloads an S3 object using a presigned URL. + *

+ *

+ * This operation uses a presigned URL that contains all necessary authentication information, eliminating the need for AWS + * credentials at request time. The presigned URL must be valid and not expired. + *

+ *
+ *
Range Requests
+ *
+ *

+ * Supports partial object downloads using HTTP Range headers. Specify the range parameter + * in the request to download only a portion of the object (e.g., "bytes=0-1023"). + *

+ *
+ *
Storage Classes
+ *
+ *

+ * If the object is stored in S3 Glacier Flexible Retrieval, S3 Glacier Deep Archive, + * or S3 Intelligent-Tiering Archive tiers, the object must be restored before retrieval. + * Otherwise, this operation returns an InvalidObjectState error. + *

+ *
+ *
+ * + * @param request The presigned URL request containing the URL and optional range parameters + * @param responseTransformer Transforms the response to the desired return type. See + * {@link software.amazon.awssdk.core.sync.ResponseTransformer} for pre-built implementations like + * downloading to a file or converting to bytes. + * @param The type of the transformed response + * @return The transformed result of the ResponseTransformer + * @throws software.amazon.awssdk.services.s3.model.NoSuchKeyException The specified object does not exist + * @throws software.amazon.awssdk.services.s3.model.InvalidObjectStateException Object is archived and must be restored before + * retrieval + * @throws software.amazon.awssdk.core.exception.SdkClientException If any client side error occurs such as + * network failures or invalid presigned URL + * @throws S3Exception Base class for all S3 service exceptions. + * Unknown exceptions will be thrown as an + * instance of this type. + */ + default ReturnT getObject(PresignedUrlGetObjectRequest request, + ResponseTransformer responseTransformer) throws NoSuchKeyException, + InvalidObjectStateException, + SdkClientException, + S3Exception { + throw new UnsupportedOperationException(); + } + + //TODO: Add other getObject method flavors +} diff --git a/services/s3/src/main/java/software/amazon/awssdk/services/s3/presignedurl/model/PresignedUrlGetObjectRequest.java b/services/s3/src/main/java/software/amazon/awssdk/services/s3/presignedurl/model/PresignedUrlGetObjectRequest.java new file mode 100644 index 000000000000..20cd9c89d8a2 --- /dev/null +++ b/services/s3/src/main/java/software/amazon/awssdk/services/s3/presignedurl/model/PresignedUrlGetObjectRequest.java @@ -0,0 +1,167 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.services.s3.presignedurl.model; + +import java.net.URL; +import java.util.Objects; +import java.util.function.Consumer; +import software.amazon.awssdk.annotations.SdkPublicApi; +import software.amazon.awssdk.utils.ToString; +import software.amazon.awssdk.utils.Validate; +import software.amazon.awssdk.utils.builder.CopyableBuilder; +import software.amazon.awssdk.utils.builder.ToCopyableBuilder; + +/** + * Request object for performing GetObject operations using a presigned URL. + */ +@SdkPublicApi +public final class PresignedUrlGetObjectRequest implements ToCopyableBuilder { + private final URL presignedUrl; + private final String range; + + private PresignedUrlGetObjectRequest(BuilderImpl builder) { + this.presignedUrl = builder.presignedUrl; + this.range = builder.range; + } + + /** + *

+ * The presigned URL for the S3 object. This URL contains all necessary authentication information and can be used to download + * the object without additional credentials. + *

+ * Note: Presigned URLs have a limited lifetime and will expire after the + * specified duration. Ensure the URL is used before expiration. + * + * @return The presigned URL for the S3 object + */ + public URL presignedUrl() { + return presignedUrl; + } + + /** + *

+ * Specifies the byte range of an object. For more information about the HTTP Range header, see + * + * https://www.rfc-editor.org/rfc/rfc9110.html#name-range. + *

+ * Note: Amazon S3 doesn't support retrieving multiple ranges of data per GET request. + * + * @return The HTTP Range header value, or null if not specified. + */ + public String range() { + return range; + } + + @Override + public Builder toBuilder() { + return new BuilderImpl(this); + } + + public static Builder builder() { + return new BuilderImpl(); + } + + /** + * Create a {@code PresignedUrlGetObjectRequest} instance with the given configuration + * + * @param builderConsumer the consumer that will configure the builder + * @return a {@code PresignedUrlGetObjectRequest} instance + */ + public static PresignedUrlGetObjectRequest builder(Consumer builderConsumer) { + return builder().applyMutation(builderConsumer).build(); + } + + public static Class serializableBuilderClass() { + return BuilderImpl.class; + } + + @Override + public int hashCode() { + int hashCode = 1; + hashCode = 31 * hashCode + Objects.hashCode(presignedUrl()); + hashCode = 31 * hashCode + Objects.hashCode(range()); + return hashCode; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null || getClass() != obj.getClass()) { + return false; + } + PresignedUrlGetObjectRequest other = (PresignedUrlGetObjectRequest) obj; + return Objects.equals(presignedUrl(), other.presignedUrl()) && + Objects.equals(range(), other.range()); + } + + @Override + public String toString() { + return ToString.builder("PresignedUrlGetObjectRequest") + .add("PresignedUrl", presignedUrl()) + .add("Range", range()) + .build(); + } + + public interface Builder extends CopyableBuilder { + /** + * Sets the presigned URL for the S3 object. + * @param presignedUrl + * @return Returns a reference to this object so that method calls can be chained together. + */ + Builder presignedUrl(URL presignedUrl); + + /** + * Specifies the byte range of an object. + * @param range The HTTP Range header value (e.g., "bytes=0-1023") + * @return Returns a reference to this object so that method calls can be chained together. + */ + Builder range(String range); + } + + static final class BuilderImpl implements Builder { + private URL presignedUrl; + private String range; + + private BuilderImpl() { + } + + private BuilderImpl(PresignedUrlGetObjectRequest presignedUrlGetObjectRequest) { + presignedUrl(presignedUrlGetObjectRequest.presignedUrl()); + range(presignedUrlGetObjectRequest.range()); + } + + @Override + public Builder presignedUrl(URL presignedUrl) { + this.presignedUrl = presignedUrl; + return this; + } + + @Override + public Builder range(String range) { + this.range = range; + return this; + } + + @Override + public PresignedUrlGetObjectRequest build() { + Validate.paramNotNull(presignedUrl, "presignedUrl"); + return new PresignedUrlGetObjectRequest(this); + } + } +} diff --git a/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/presignedurl/model/PresignedUrlGetObjectRequestWrapperTest.java b/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/presignedurl/model/PresignedUrlGetObjectRequestWrapperTest.java new file mode 100644 index 000000000000..8334050fa7c8 --- /dev/null +++ b/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/presignedurl/model/PresignedUrlGetObjectRequestWrapperTest.java @@ -0,0 +1,94 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.services.s3.internal.presignedurl.model; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.net.URL; +import java.util.List; +import java.util.Map; +import nl.jqno.equalsverifier.EqualsVerifier; +import org.junit.jupiter.api.Test; +import software.amazon.awssdk.core.SdkField; +import software.amazon.awssdk.core.protocol.MarshallLocation; + + +class PresignedUrlGetObjectRequestWrapperTest { + @Test + void equalsAndHashCode_shouldFollowContract() { + EqualsVerifier.forClass(PresignedUrlGetObjectRequestWrapper.class) + .withRedefinedSuperclass() + .verify(); + } + + @Test + void basicProperties_shouldWork() throws Exception { + PresignedUrlGetObjectRequestWrapper request = PresignedUrlGetObjectRequestWrapper.builder() + .url(new URL("https://example.com")) + .range("bytes=0-100") + .build(); + + assertThat(request.url()).isEqualTo(new URL("https://example.com")); + assertThat(request.range()).isEqualTo("bytes=0-100"); + } + + @Test + void sdkFields_shouldReturnExpectedFields() throws Exception { + PresignedUrlGetObjectRequestWrapper request = PresignedUrlGetObjectRequestWrapper.builder() + .url(new URL("https://example.com")) + .range("bytes=0-100") + .build(); + + List> fields = request.sdkFields(); + + assertThat(fields).hasSize(1); + assertThat(fields.get(0).memberName()).isEqualTo("Range"); + assertThat(fields.get(0).location()).isEqualTo(MarshallLocation.HEADER); + + // Assert that range value is present + Object rangeValue = fields.get(0).getValueOrDefault(request); + assertThat(rangeValue).isNotNull(); + assertThat(rangeValue).isEqualTo("bytes=0-100"); + } + + @Test + void sdkFieldNameToField_shouldReturnExpectedMapping() throws Exception { + PresignedUrlGetObjectRequestWrapper request = PresignedUrlGetObjectRequestWrapper.builder() + .url(new URL("https://example.com")) + .build(); + + Map> fieldMap = request.sdkFieldNameToField(); + + assertThat(fieldMap) + .hasSize(1) + .containsKey("Range"); + assertThat(fieldMap.get("Range").memberName()).isEqualTo("Range"); + } + + @Test + void rangeField_shouldMarshalCorrectly() throws Exception { + PresignedUrlGetObjectRequestWrapper request = PresignedUrlGetObjectRequestWrapper.builder() + .url(new URL("https://example.com")) + .range("bytes=0-1023") + .build(); + + // Test that the SdkField can extract the range value + SdkField rangeField = request.sdkFields().get(0); + Object extractedValue = rangeField.getValueOrDefault(request); + + assertThat(extractedValue).isEqualTo("bytes=0-1023"); + } +} diff --git a/services/s3/src/test/java/software/amazon/awssdk/services/s3/presignedurl/model/PresignedUrlGetObjectRequestTest.java b/services/s3/src/test/java/software/amazon/awssdk/services/s3/presignedurl/model/PresignedUrlGetObjectRequestTest.java new file mode 100644 index 000000000000..14048f2fe1a7 --- /dev/null +++ b/services/s3/src/test/java/software/amazon/awssdk/services/s3/presignedurl/model/PresignedUrlGetObjectRequestTest.java @@ -0,0 +1,115 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.services.s3.presignedurl.model; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.net.URL; +import nl.jqno.equalsverifier.EqualsVerifier; +import org.junit.jupiter.api.Test; + +class PresignedUrlGetObjectRequestTest { + + @Test + void equalsAndHashCode_shouldFollowContract() { + EqualsVerifier.forClass(PresignedUrlGetObjectRequest.class) + .verify(); + } + + @Test + void builder_shouldCreateRequestWithAllFields() throws Exception { + URL url = new URL("https://example.com"); + PresignedUrlGetObjectRequest request = PresignedUrlGetObjectRequest.builder() + .presignedUrl(url) + .range("bytes=0-100") + .build(); + + assertThat(request.presignedUrl()).isEqualTo(url); + assertThat(request.range()).isEqualTo("bytes=0-100"); + } + + @Test + void builder_shouldCreateRequestWithOnlyRequiredFields() throws Exception { + URL url = new URL("https://example.com"); + PresignedUrlGetObjectRequest request = PresignedUrlGetObjectRequest.builder() + .presignedUrl(url) + .build(); + + assertThat(request.presignedUrl()).isEqualTo(url); + assertThat(request.range()).isNull(); + } + + @Test + void toBuilder_shouldCreateBuilderFromExistingRequest() throws Exception { + URL url = new URL("https://example.com"); + PresignedUrlGetObjectRequest original = PresignedUrlGetObjectRequest.builder() + .presignedUrl(url) + .range("bytes=0-100") + .build(); + + PresignedUrlGetObjectRequest copy = original.toBuilder().build(); + + assertThat(copy.presignedUrl()).isEqualTo(original.presignedUrl()); + assertThat(copy.range()).isEqualTo(original.range()); + } + + @Test + void toBuilder_shouldAllowModification() throws Exception { + URL url1 = new URL("https://example.com"); + URL url2 = new URL("https://other.com"); + PresignedUrlGetObjectRequest original = PresignedUrlGetObjectRequest.builder() + .presignedUrl(url1) + .range("bytes=0-100") + .build(); + + PresignedUrlGetObjectRequest modified = original.toBuilder() + .presignedUrl(url2) + .range("bytes=200-300") + .build(); + + assertThat(modified.presignedUrl()).isEqualTo(url2); + assertThat(modified.range()).isEqualTo("bytes=200-300"); + // Original unchanged + assertThat(original.presignedUrl()).isEqualTo(url1); + assertThat(original.range()).isEqualTo("bytes=0-100"); + } + + @Test + void toString_shouldContainActualFieldValues() throws Exception { + URL url = new URL("https://example.com"); + String range = "bytes=0-100"; + + PresignedUrlGetObjectRequest request = PresignedUrlGetObjectRequest.builder() + .presignedUrl(url) + .range(range) + .build(); + + String result = request.toString(); + + assertThat(result) + .isNotNull() + .isNotEmpty() + .contains(request.presignedUrl().toString()) + .contains(request.range()); + + } + + @Test + void serializableBuilderClass_shouldReturnCorrectClass() { + assertThat(PresignedUrlGetObjectRequest.serializableBuilderClass()) + .isEqualTo(PresignedUrlGetObjectRequest.BuilderImpl.class); + } +} From 451a70bed0ff222aa065f48900dfac0dcb0e8d1a Mon Sep 17 00:00:00 2001 From: jencymaryjoseph <35571282+jencymaryjoseph@users.noreply.github.com> Date: Fri, 4 Jul 2025 10:04:08 -0700 Subject: [PATCH 02/30] Add custom marshaller for pre-signed URL GET operations (#6231) * presigned url get object request custom marshaller * used protocolFactory and fixed sonar cube issues * removed unused test method, removed overriden method --- ...resignedUrlGetObjectRequestMarshaller.java | 76 ++++++++ ...gnedUrlGetObjectRequestMarshallerTest.java | 181 ++++++++++++++++++ 2 files changed, 257 insertions(+) create mode 100644 services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/presignedurl/PresignedUrlGetObjectRequestMarshaller.java create mode 100644 services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/presignedurl/PresignedUrlGetObjectRequestMarshallerTest.java diff --git a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/presignedurl/PresignedUrlGetObjectRequestMarshaller.java b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/presignedurl/PresignedUrlGetObjectRequestMarshaller.java new file mode 100644 index 000000000000..76e1500f2d54 --- /dev/null +++ b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/presignedurl/PresignedUrlGetObjectRequestMarshaller.java @@ -0,0 +1,76 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.services.s3.internal.presignedurl; + +import java.net.URI; +import software.amazon.awssdk.annotations.SdkInternalApi; +import software.amazon.awssdk.core.exception.SdkClientException; +import software.amazon.awssdk.core.runtime.transform.Marshaller; +import software.amazon.awssdk.http.SdkHttpFullRequest; +import software.amazon.awssdk.http.SdkHttpMethod; +import software.amazon.awssdk.protocols.core.OperationInfo; +import software.amazon.awssdk.protocols.core.ProtocolMarshaller; +import software.amazon.awssdk.protocols.xml.AwsXmlProtocolFactory; +import software.amazon.awssdk.services.s3.internal.presignedurl.model.PresignedUrlGetObjectRequestWrapper; +import software.amazon.awssdk.utils.Validate; + +/** + * {@link PresignedUrlGetObjectRequestWrapper} Marshaller + * + *

+ * Marshalls presigned URL requests by using the complete URL directly and adding optional Range headers. + * Unlike regular S3 marshalers, this preserves all embedded authentication parameters in the presigned URL. + *

+ */ +@SdkInternalApi +public class PresignedUrlGetObjectRequestMarshaller implements Marshaller { + private static final OperationInfo SDK_OPERATION_BINDING = OperationInfo.builder() + .requestUri("").httpMethod(SdkHttpMethod.GET).hasExplicitPayloadMember(false).hasPayloadMembers(false) + .putAdditionalMetadata(AwsXmlProtocolFactory.ROOT_MARSHALL_LOCATION_ATTRIBUTE, null) + .putAdditionalMetadata(AwsXmlProtocolFactory.XML_NAMESPACE_ATTRIBUTE, null).build(); + + private final AwsXmlProtocolFactory protocolFactory; + + public PresignedUrlGetObjectRequestMarshaller(AwsXmlProtocolFactory protocolFactory) { + this.protocolFactory = protocolFactory; + } + + /** + * Marshalls the presigned URL request into an HTTP GET request. + * + * @param presignedUrlGetObjectRequestWrapper the request to marshall + * @return HTTP request ready for execution + * @throws SdkClientException if URL conversion fails + */ + @Override + public SdkHttpFullRequest marshall(PresignedUrlGetObjectRequestWrapper presignedUrlGetObjectRequestWrapper) { + Validate.paramNotNull(presignedUrlGetObjectRequestWrapper, "presignedUrlGetObjectRequestWrapper"); + try { + ProtocolMarshaller protocolMarshaller = protocolFactory + .createProtocolMarshaller(SDK_OPERATION_BINDING); + URI presignedUri = presignedUrlGetObjectRequestWrapper.url().toURI(); + + return protocolMarshaller.marshall(presignedUrlGetObjectRequestWrapper) + .toBuilder() + .uri(presignedUri) + .build(); + } catch (Exception e) { + throw SdkClientException.builder() + .message("Unable to marshall pre-signed URL Request: " + e.getMessage()) + .cause(e).build(); + } + } +} \ No newline at end of file diff --git a/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/presignedurl/PresignedUrlGetObjectRequestMarshallerTest.java b/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/presignedurl/PresignedUrlGetObjectRequestMarshallerTest.java new file mode 100644 index 000000000000..2311017a29e5 --- /dev/null +++ b/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/presignedurl/PresignedUrlGetObjectRequestMarshallerTest.java @@ -0,0 +1,181 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.services.s3.internal.presignedurl; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; +import java.net.URL; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.NullAndEmptySource; +import org.junit.jupiter.params.provider.ValueSource; +import software.amazon.awssdk.core.exception.SdkClientException; +import software.amazon.awssdk.http.SdkHttpFullRequest; +import software.amazon.awssdk.http.SdkHttpMethod; +import software.amazon.awssdk.protocols.core.OperationInfo; +import software.amazon.awssdk.protocols.core.ProtocolMarshaller; +import software.amazon.awssdk.protocols.xml.AwsXmlProtocolFactory; +import software.amazon.awssdk.services.s3.internal.presignedurl.model.PresignedUrlGetObjectRequestWrapper; + +class PresignedUrlGetObjectRequestMarshallerTest { + + private PresignedUrlGetObjectRequestMarshaller marshaller; + private AwsXmlProtocolFactory mockProtocolFactory; + private ProtocolMarshaller mockProtocolMarshaller; + private URL testUrl; + + @BeforeEach + void setUp() throws Exception { + mockProtocolFactory = mock(AwsXmlProtocolFactory.class); + mockProtocolMarshaller = mock(ProtocolMarshaller.class); + when(mockProtocolFactory.createProtocolMarshaller(any(OperationInfo.class))) + .thenReturn(mockProtocolMarshaller); + marshaller = new PresignedUrlGetObjectRequestMarshaller(mockProtocolFactory); + + testUrl = new URL("https://test-bucket.s3.us-east-1.amazonaws.com/test-key?" + + "X-Amz-Date=20231215T000000Z&" + + "X-Amz-Signature=example-signature&" + + "X-Amz-Algorithm=AWS4-HMAC-SHA256&" + + "X-Amz-SignedHeaders=host&" + + "X-Amz-Security-Token=xxx&" + + "X-Amz-Credential=EXAMPLE12345678901234%2F20231215%2Fus-east-1%2Fs3%2Faws4_request&" + + "X-Amz-Expires=3600"); + } + + @Test + void marshall_withBasicRequest_shouldCreateCorrectHttpRequest() throws Exception { + // Setup the mock marshaller to return a properly configured request + SdkHttpFullRequest baseRequest = SdkHttpFullRequest.builder() + .method(SdkHttpMethod.GET) + .protocol("https") + .host("example.com") + .build(); + when(mockProtocolMarshaller.marshall(any(PresignedUrlGetObjectRequestWrapper.class))) + .thenReturn(baseRequest); + + PresignedUrlGetObjectRequestWrapper request = PresignedUrlGetObjectRequestWrapper.builder() + .url(testUrl) + .build(); + SdkHttpFullRequest result = marshaller.marshall(request); + + // Verify HTTP method and URI components + assertThat(result.method()).isEqualTo(SdkHttpMethod.GET); + assertThat(result.getUri()) + .satisfies(uri -> { + assertThat(uri.getScheme()).isEqualTo("https"); + assertThat(uri.getHost()).isEqualTo("test-bucket.s3.us-east-1.amazonaws.com"); + assertThat(uri.getPath()).isEqualTo("/test-key"); + }); + + // Verify query parameters are preserved + assertThat(result.getUri().getQuery()) + .contains("X-Amz-Date=20231215T000000Z") + .contains("X-Amz-Signature=example-signature") + .contains("X-Amz-Algorithm=AWS4-HMAC-SHA256") + .contains("X-Amz-SignedHeaders=host") + .contains("X-Amz-Security-Token=xxx") + .contains("X-Amz-Credential=EXAMPLE12345678901234") + .contains("X-Amz-Expires=3600"); + + assertThat(result.headers()).doesNotContainKey("Range"); + } + + @ParameterizedTest + @ValueSource(strings = { + "bytes=0-100", // First 101 bytes + "bytes=100-", // From byte 100 to end + "bytes=-100", // Last 100 bytes + "bytes=0-0", // Single byte + "bytes=100-200" // Specific range + }) + void marshall_withValidRangeFormats_shouldAddRangeHeader(String rangeValue) throws Exception { + // Setup the mock marshaller to return a request with the Range header already set + SdkHttpFullRequest baseRequest = SdkHttpFullRequest.builder() + .method(SdkHttpMethod.GET) + .protocol("https") + .host("example.com") + .putHeader("Range", rangeValue) // Add the Range header to the mock response + .build(); + + when(mockProtocolMarshaller.marshall(any(PresignedUrlGetObjectRequestWrapper.class))) + .thenReturn(baseRequest); + + PresignedUrlGetObjectRequestWrapper request = PresignedUrlGetObjectRequestWrapper.builder() + .url(testUrl) + .range(rangeValue) + .build(); + + SdkHttpFullRequest result = marshaller.marshall(request); + + // Verify the Range header is preserved + assertThat(result.headers()) + .containsKey("Range") + .satisfies(headers -> assertThat(headers.get("Range")).contains(rangeValue)); + } + + @ParameterizedTest + @NullAndEmptySource + void marshall_withNullOrEmptyRange_shouldNotAddRangeHeader(String rangeValue) throws Exception { + // Setup the mock marshaller to return a properly configured request + SdkHttpFullRequest baseRequest = SdkHttpFullRequest.builder() + .method(SdkHttpMethod.GET) + .protocol("https") + .host("example.com") + .build(); + when(mockProtocolMarshaller.marshall(any(PresignedUrlGetObjectRequestWrapper.class))) + .thenReturn(baseRequest); + + PresignedUrlGetObjectRequestWrapper request = PresignedUrlGetObjectRequestWrapper.builder() + .url(testUrl) + .range(rangeValue) + .build(); + SdkHttpFullRequest result = marshaller.marshall(request); + + assertThat(result.headers()).doesNotContainKey("Range"); + } + + @Test + void marshall_withNullRequest_shouldThrowException() { + assertThatThrownBy(() -> marshaller.marshall(null)) + .isInstanceOf(NullPointerException.class) + .hasMessageContaining("presignedUrlGetObjectRequestWrapper must not be null"); + } + + @Test + void marshall_withMalformedUrl_shouldThrowSdkClientException() throws Exception { + // Setup the mock marshaller to return a properly configured request + SdkHttpFullRequest baseRequest = SdkHttpFullRequest.builder() + .method(SdkHttpMethod.GET) + .protocol("https") + .host("example.com") + .build(); + when(mockProtocolMarshaller.marshall(any(PresignedUrlGetObjectRequestWrapper.class))) + .thenReturn(baseRequest); + + URL malformedUrl = new URL("https", "test-bucket.s3.us-east-1.amazonaws.com", -1, "/test key with spaces"); + PresignedUrlGetObjectRequestWrapper request = PresignedUrlGetObjectRequestWrapper.builder() + .url(malformedUrl) + .build(); + + assertThatThrownBy(() -> marshaller.marshall(request)) + .isInstanceOf(SdkClientException.class) + .hasMessageContaining("Unable to marshall pre-signed URL Request"); + } +} \ No newline at end of file From 563237b8e1fa878718b78a7f36a15f6604f2d03f Mon Sep 17 00:00:00 2001 From: jencymaryjoseph <35571282+jencymaryjoseph@users.noreply.github.com> Date: Thu, 10 Jul 2025 11:02:33 -0700 Subject: [PATCH 03/30] Add DefaultPresignedUrlManager getObject implementation for S3 pre-signed URL (#6241) * getobject implementation and tests * corrected the getObject params in test: * fixed build failure-dependency issue with Retrystrategy in test * fixed sonarcube issues * addressed sonarcube issue * edited javadocs, added invalidurl test --- .../DefaultPresignedUrlManager.java | 122 ++++++ .../DefaultPresignedUrlManagerTest.java | 378 ++++++++++++++++++ 2 files changed, 500 insertions(+) create mode 100644 services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/presignedurl/DefaultPresignedUrlManager.java create mode 100644 services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/presignedurl/DefaultPresignedUrlManagerTest.java diff --git a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/presignedurl/DefaultPresignedUrlManager.java b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/presignedurl/DefaultPresignedUrlManager.java new file mode 100644 index 000000000000..3527b4d830d4 --- /dev/null +++ b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/presignedurl/DefaultPresignedUrlManager.java @@ -0,0 +1,122 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.services.s3.internal.presignedurl; + +import static software.amazon.awssdk.core.client.config.SdkClientOption.SIGNER_OVERRIDDEN; + +import java.util.Collections; +import java.util.List; +import java.util.Optional; +import software.amazon.awssdk.annotations.SdkInternalApi; +import software.amazon.awssdk.awscore.exception.AwsServiceException; +import software.amazon.awssdk.awscore.internal.AwsProtocolMetadata; +import software.amazon.awssdk.core.client.config.SdkAdvancedClientOption; +import software.amazon.awssdk.core.client.config.SdkClientConfiguration; +import software.amazon.awssdk.core.client.config.SdkClientOption; +import software.amazon.awssdk.core.client.handler.ClientExecutionParams; +import software.amazon.awssdk.core.client.handler.SyncClientHandler; +import software.amazon.awssdk.core.exception.SdkClientException; +import software.amazon.awssdk.core.http.HttpResponseHandler; +import software.amazon.awssdk.core.interceptor.SdkInternalExecutionAttribute; +import software.amazon.awssdk.core.metrics.CoreMetric; +import software.amazon.awssdk.core.signer.NoOpSigner; +import software.amazon.awssdk.core.sync.ResponseTransformer; +import software.amazon.awssdk.metrics.MetricCollector; +import software.amazon.awssdk.metrics.MetricPublisher; +import software.amazon.awssdk.metrics.NoOpMetricCollector; +import software.amazon.awssdk.protocols.xml.AwsS3ProtocolFactory; +import software.amazon.awssdk.protocols.xml.XmlOperationMetadata; +import software.amazon.awssdk.services.s3.internal.presignedurl.model.PresignedUrlGetObjectRequestWrapper; +import software.amazon.awssdk.services.s3.model.GetObjectResponse; +import software.amazon.awssdk.services.s3.model.InvalidObjectStateException; +import software.amazon.awssdk.services.s3.model.NoSuchKeyException; +import software.amazon.awssdk.services.s3.model.S3Exception; +import software.amazon.awssdk.services.s3.presignedurl.PresignedUrlManager; +import software.amazon.awssdk.services.s3.presignedurl.model.PresignedUrlGetObjectRequest; + +/** + * Default implementation of {@link PresignedUrlManager} for executing S3 operations using presigned URLs. + */ +@SdkInternalApi +public final class DefaultPresignedUrlManager implements PresignedUrlManager { + + private final SyncClientHandler clientHandler; + private final AwsS3ProtocolFactory protocolFactory; + private final SdkClientConfiguration clientConfiguration; + private final AwsProtocolMetadata protocolMetadata; + + public DefaultPresignedUrlManager(SyncClientHandler clientHandler, + AwsS3ProtocolFactory protocolFactory, + SdkClientConfiguration clientConfiguration, + AwsProtocolMetadata protocolMetadata) { + this.clientHandler = clientHandler; + this.protocolFactory = protocolFactory; + this.clientConfiguration = clientConfiguration; + this.protocolMetadata = protocolMetadata; + } + + @Override + public ReturnT getObject(PresignedUrlGetObjectRequest presignedUrlGetObjectRequest, + ResponseTransformer responseTransformer) + throws NoSuchKeyException, InvalidObjectStateException, + AwsServiceException, SdkClientException, S3Exception { + + HttpResponseHandler responseHandler = protocolFactory.createResponseHandler( + GetObjectResponse::builder, new XmlOperationMetadata().withHasStreamingSuccessResponse(true)); + + HttpResponseHandler errorResponseHandler = protocolFactory.createErrorResponseHandler(); + + PresignedUrlGetObjectRequestWrapper internalRequest = PresignedUrlGetObjectRequestWrapper.builder() + .url(presignedUrlGetObjectRequest.presignedUrl()) + .range(presignedUrlGetObjectRequest.range()) + .build(); + + SdkClientConfiguration updatedClientConfiguration = updateSdkClientConfiguration(this.clientConfiguration); + List metricPublishers = Optional.ofNullable( + updatedClientConfiguration.option(SdkClientOption.METRIC_PUBLISHERS)) + .orElse(Collections.emptyList()); + MetricCollector apiCallMetricCollector = metricPublishers.isEmpty() ? + NoOpMetricCollector.create() : MetricCollector.create("ApiCall"); + try { + apiCallMetricCollector.reportMetric(CoreMetric.SERVICE_ID, "S3"); + //TODO: Discuss if we need to change OPERATION_NAME as part of Surface API Review + apiCallMetricCollector.reportMetric(CoreMetric.OPERATION_NAME, "GetObject"); + + return clientHandler.execute( + new ClientExecutionParams() + .withOperationName("PresignedUrlGetObject") + .withProtocolMetadata(protocolMetadata) + .withResponseHandler(responseHandler) + .withErrorResponseHandler(errorResponseHandler) + .withRequestConfiguration(updatedClientConfiguration) + .withInput(internalRequest) + .withMetricCollector(apiCallMetricCollector) + // TODO: Deprecate IS_DISCOVERED_ENDPOINT, use new SKIP_ENDPOINT_RESOLUTION for better semantics + .putExecutionAttribute(SdkInternalExecutionAttribute.IS_DISCOVERED_ENDPOINT, true) + .withMarshaller(new PresignedUrlGetObjectRequestMarshaller(protocolFactory)), responseTransformer); + } finally { + metricPublishers.forEach(p -> p.publish(apiCallMetricCollector.collect())); + } + } + + private SdkClientConfiguration updateSdkClientConfiguration(SdkClientConfiguration clientConfiguration) { + SdkClientConfiguration.Builder configBuilder = clientConfiguration.toBuilder(); + configBuilder.option(SdkAdvancedClientOption.SIGNER, new NoOpSigner()); + configBuilder.option(SIGNER_OVERRIDDEN, true); + return configBuilder.build(); + } + +} diff --git a/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/presignedurl/DefaultPresignedUrlManagerTest.java b/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/presignedurl/DefaultPresignedUrlManagerTest.java new file mode 100644 index 000000000000..ff6266689927 --- /dev/null +++ b/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/presignedurl/DefaultPresignedUrlManagerTest.java @@ -0,0 +1,378 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.services.s3.internal.presignedurl; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.net.URI; +import java.net.URL; +import java.nio.charset.StandardCharsets; +import java.util.Collections; +import java.util.function.Consumer; +import java.util.stream.Stream; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import org.mockito.ArgumentCaptor; +import software.amazon.awssdk.awscore.retry.AwsRetryStrategy; +import software.amazon.awssdk.core.exception.SdkClientException; +import software.amazon.awssdk.metrics.MetricCollection; +import software.amazon.awssdk.metrics.MetricPublisher; +import software.amazon.awssdk.core.metrics.CoreMetric; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.atLeastOnce; +import static org.mockito.Mockito.any; +import software.amazon.awssdk.auth.credentials.AnonymousCredentialsProvider; +import software.amazon.awssdk.awscore.client.config.AwsAdvancedClientOption; +import software.amazon.awssdk.awscore.client.config.AwsClientOption; +import software.amazon.awssdk.awscore.client.handler.AwsSyncClientHandler; +import software.amazon.awssdk.awscore.internal.AwsProtocolMetadata; +import software.amazon.awssdk.awscore.internal.AwsServiceProtocol; +import software.amazon.awssdk.core.ResponseInputStream; +import software.amazon.awssdk.core.client.config.SdkAdvancedClientOption; +import software.amazon.awssdk.core.client.config.SdkClientConfiguration; +import software.amazon.awssdk.core.client.config.SdkClientOption; +import software.amazon.awssdk.core.client.handler.SyncClientHandler; +import software.amazon.awssdk.core.sync.ResponseTransformer; +import software.amazon.awssdk.http.AbortableInputStream; +import software.amazon.awssdk.http.HttpExecuteResponse; +import software.amazon.awssdk.http.SdkHttpFullResponse; +import software.amazon.awssdk.http.SdkHttpMethod; +import software.amazon.awssdk.http.SdkHttpRequest; +import software.amazon.awssdk.protocols.core.ExceptionMetadata; +import software.amazon.awssdk.protocols.xml.AwsS3ProtocolFactory; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.s3.model.GetObjectResponse; +import software.amazon.awssdk.services.s3.model.InvalidObjectStateException; +import software.amazon.awssdk.services.s3.model.NoSuchKeyException; +import software.amazon.awssdk.services.s3.model.S3Exception; +import software.amazon.awssdk.services.s3.presignedurl.model.PresignedUrlGetObjectRequest; +import software.amazon.awssdk.testutils.service.http.MockSyncHttpClient; +import software.amazon.awssdk.utils.IoUtils; + +/** + * Tests for {@link DefaultPresignedUrlManager} using MockSyncHttpClient to verify HTTP interactions. + */ + class DefaultPresignedUrlManagerTest { + + private static final String TEST_CONTENT = "test-content"; + private static final URI DEFAULT_ENDPOINT = URI.create("https://defaultendpoint.com"); + private static final String TEST_URL = "https://test-bucket.s3.us-east-1.amazonaws.com/test-key?" + + "X-Amz-Date=20250707T000000Z&" + + "X-Amz-Signature=test-signature-value&" + + "X-Amz-Algorithm=AWS4-HMAC-SHA256&" + + "X-Amz-SignedHeaders=host&" + + "X-Amz-Security-Token=test-session-token&" + + "X-Amz-Credential=AKIAIOSFODNN7EXAMPLE%2F20250707%2Fus-east-1%2Fs3%2Faws4_request&" + + "X-Amz-Expires=86400"; + private MockSyncHttpClient mockHttpClient; + private DefaultPresignedUrlManager presignedUrlManager; + private URL testPresignedUrl; + private PresignedUrlGetObjectRequest testRequest; + private AwsProtocolMetadata protocolMetadata; + private AwsS3ProtocolFactory protocolFactory; + private SyncClientHandler clientHandler; + private SdkClientConfiguration clientConfiguration; + + @BeforeEach + void setUp() throws Exception { + mockHttpClient = new MockSyncHttpClient(); + testPresignedUrl = new URL(TEST_URL); + testRequest = PresignedUrlGetObjectRequest.builder() + .presignedUrl(testPresignedUrl) + .build(); + + clientConfiguration = getDefaultSdkConfigs(); + protocolMetadata = AwsProtocolMetadata.builder() + .serviceProtocol(AwsServiceProtocol.REST_XML) + .build(); + protocolFactory = initProtocolFactory(clientConfiguration); + clientHandler = new AwsSyncClientHandler(clientConfiguration); + + presignedUrlManager = new DefaultPresignedUrlManager( + clientHandler, protocolFactory, clientConfiguration, protocolMetadata); + } + + private static Stream httpResponseTestCases() { + return Stream.of( + Arguments.of( + "Success response", + createSuccessResponse(), + true, + null + ), + Arguments.of( + "404 Not Found response", + createErrorResponse(404, "NoSuchKey", "The specified key does not exist."), + false, + NoSuchKeyException.class + ), + Arguments.of( + "403 Invalid Object State response", + createErrorResponse(403, "InvalidObjectState", "The operation is not valid for the object's storage class."), + false, + InvalidObjectStateException.class + ), + Arguments.of( + "Generic error response", + createErrorResponse(500, "InternalError", "We encountered an internal error. Please try again."), + false, + S3Exception.class + ) + ); + } + + private static Stream requestConfigurationTestCases() { + return Stream.of( + Arguments.of( + "Basic request", + (Consumer) builder -> + builder.presignedUrl(createTestUrl()), + null + ), + Arguments.of( + "Request with range header", + (Consumer) builder -> + builder.presignedUrl(createTestUrl()).range("bytes=0-1024"), + "bytes=0-1024" + ) + ); + } + + @ParameterizedTest(name = "{0}") + @MethodSource("httpResponseTestCases") + void given_PresignedUrlManager_when_GetObjectWithDifferentHttpResponses_then_ShouldHandleSuccessAndErrorsCorrectly(String testName, + HttpExecuteResponse response, + boolean expectSuccess, + Class expectedExceptionType) { + mockHttpClient.stubNextResponse(response); + if (expectSuccess) { + assertSuccessfulGetObject(testRequest); + } else { + assertThatThrownBy(() -> presignedUrlManager.getObject(testRequest, ResponseTransformer.toInputStream())) + .isInstanceOf(expectedExceptionType); + } + } + + @ParameterizedTest(name = "{0}") + @MethodSource("requestConfigurationTestCases") + void given_PresignedUrlManager_when_GetObjectWithDifferentRequestConfigurations_then_ShouldSetCorrectHeaders(String testName, + Consumer requestCustomizer, + String expectedRangeHeader) throws IOException { + mockHttpClient.stubNextResponse(createSuccessResponse()); + PresignedUrlGetObjectRequest.Builder builder = PresignedUrlGetObjectRequest.builder(); + requestCustomizer.accept(builder); + PresignedUrlGetObjectRequest request = builder.build(); + ResponseInputStream result = presignedUrlManager.getObject(request, ResponseTransformer.toInputStream()); + assertThat(result).isNotNull(); + String content = IoUtils.toUtf8String(result); + assertThat(content).isEqualTo(TEST_CONTENT); + SdkHttpRequest lastRequest = mockHttpClient.getLastRequest(); + assertThat(lastRequest.method()).isEqualTo(SdkHttpMethod.GET); + if (expectedRangeHeader != null) { + assertThat(lastRequest.firstMatchingHeader("Range")).isPresent() + .contains(expectedRangeHeader); + } + } + + private static Stream additionalTestCases() { + return Stream.of( + Arguments.of( + "Custom transformer test", + "CUSTOM_TRANSFORMER" + ), + Arguments.of( + "Resolved endpoint verification", + "ENDPOINT_VERIFICATION" + ), + Arguments.of( + "Metrics collection test", + "METRICS_COLLECTION" + ) + ); + } + + private static Stream invalidUrlTestCases() { + return Stream.of( + Arguments.of( + "URL with spaces in path", + "https://test-bucket.s3.us-east-1.amazonaws.com/test key with spaces" + ), + Arguments.of( + "URL with invalid characters", + "https://test-bucket.s3.us-east-1.amazonaws.com/test<>key" + ), + Arguments.of( + "Malformed URL", + "not-a-valid-url" + ) + ); + } + + @ParameterizedTest(name = "{0}") + @MethodSource("additionalTestCases") + void given_PresignedUrlManager_when_ExecutingDifferentScenarios_then_ShouldBehaveCorrectly(String testName, + String testType) { + switch (testType) { + case "CUSTOM_TRANSFORMER": + mockHttpClient.stubNextResponse(createSuccessResponse()); + ResponseTransformer transformer = + (response, inputStream) -> IoUtils.toUtf8String(inputStream); + String result = presignedUrlManager.getObject(testRequest, transformer); + assertThat(result).isEqualTo(TEST_CONTENT); + break; + + case "ENDPOINT_VERIFICATION": + mockHttpClient.stubNextResponse(createSuccessResponse()); + presignedUrlManager.getObject(testRequest, ResponseTransformer.toInputStream()); + SdkHttpRequest lastRequest = mockHttpClient.getLastRequest(); + assertThat(lastRequest.getUri().toString()).startsWith(testPresignedUrl.toString().split("\\?")[0]); + String presignedUrlQuery = testPresignedUrl.getQuery(); + for (String param : presignedUrlQuery.split("&")) { + String[] keyValue = param.split("=", 2); + if (keyValue.length == 2) { + String key = keyValue[0]; + String value = keyValue[1]; + assertThat(lastRequest.getUri().toString()).contains(key + "=" + value); + } + } + break; + + case "METRICS_COLLECTION": + MetricPublisher mockPublisher = mock(MetricPublisher.class); + SdkClientConfiguration clientConfigWithMetrics = getDefaultSdkConfigs().toBuilder() + .option(SdkClientOption.METRIC_PUBLISHERS, Collections.singletonList(mockPublisher)) + .build(); + DefaultPresignedUrlManager managerWithMetrics = new DefaultPresignedUrlManager( + clientHandler, protocolFactory, clientConfigWithMetrics, protocolMetadata); + mockHttpClient.stubNextResponse(createSuccessResponse()); + managerWithMetrics.getObject(testRequest, ResponseTransformer.toInputStream()); + verify(mockPublisher, atLeastOnce()).publish(any(MetricCollection.class)); + ArgumentCaptor metricsCaptor = ArgumentCaptor.forClass(MetricCollection.class); + verify(mockPublisher).publish(metricsCaptor.capture()); + MetricCollection capturedMetrics = metricsCaptor.getValue(); + assertThat(capturedMetrics.metricValues(CoreMetric.SERVICE_ID)).contains("S3"); + assertThat(capturedMetrics.metricValues(CoreMetric.OPERATION_NAME)).contains("GetObject"); + break; + } + } + + @ParameterizedTest(name = "{0}") + @MethodSource("invalidUrlTestCases") + void given_PresignedUrlManager_when_GetObjectWithInvalidUrl_then_ShouldThrowException(String testName, String invalidUrlString) { + assertThatThrownBy(() -> { + URL invalidUrl = new URL(invalidUrlString); + PresignedUrlGetObjectRequest invalidRequest = PresignedUrlGetObjectRequest.builder() + .presignedUrl(invalidUrl) + .build(); + presignedUrlManager.getObject(invalidRequest, ResponseTransformer.toInputStream()); + }).satisfiesAnyOf( + ex -> assertThat(ex).isInstanceOf(java.net.MalformedURLException.class), + ex -> assertThat(ex).isInstanceOf(SdkClientException.class) + ); + } + + private SdkClientConfiguration getDefaultSdkConfigs() { + return SdkClientConfiguration.builder() + .option(SdkClientOption.SYNC_HTTP_CLIENT, mockHttpClient) + .option(SdkClientOption.ADDITIONAL_HTTP_HEADERS, Collections.emptyMap()) + .option(SdkClientOption.EXECUTION_INTERCEPTORS, Collections.emptyList()) + .option(SdkClientOption.RETRY_STRATEGY, AwsRetryStrategy.doNotRetry()) + .option(SdkAdvancedClientOption.USER_AGENT_PREFIX, "") + .option(SdkAdvancedClientOption.USER_AGENT_SUFFIX, "") + .option(SdkClientOption.CRC32_FROM_COMPRESSED_DATA_ENABLED, false) + .option(AwsClientOption.CREDENTIALS_IDENTITY_PROVIDER, AnonymousCredentialsProvider.create()) + .option(AwsClientOption.AWS_REGION, Region.US_EAST_2) + .option(AwsClientOption.SIGNING_REGION, Region.US_EAST_2) + .option(AwsClientOption.SERVICE_SIGNING_NAME, Region.AP_EAST_2.toString()) + .option(AwsAdvancedClientOption.ENABLE_DEFAULT_REGION_DETECTION, false) + .option(SdkClientOption.CLIENT_ENDPOINT_PROVIDER, + software.amazon.awssdk.core.ClientEndpointProvider.forEndpointOverride(DEFAULT_ENDPOINT)) + .build(); + } + + private AwsS3ProtocolFactory initProtocolFactory(SdkClientConfiguration configuration) { + return AwsS3ProtocolFactory.builder() + .registerModeledException( + ExceptionMetadata.builder().errorCode("NoSuchKey") + .exceptionBuilderSupplier(NoSuchKeyException::builder) + .httpStatusCode(404).build()) + .registerModeledException( + ExceptionMetadata.builder().errorCode("InvalidObjectState") + .exceptionBuilderSupplier(InvalidObjectStateException::builder) + .httpStatusCode(403).build()) + .clientConfiguration(configuration) + .defaultServiceExceptionSupplier(S3Exception::builder) + .build(); + } + + private static URL createTestUrl() { + try { + return new URL(TEST_URL); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + private void assertSuccessfulGetObject(PresignedUrlGetObjectRequest request) { + try { + ResponseInputStream result = presignedUrlManager.getObject(request, ResponseTransformer.toInputStream()); + assertThat(result).isNotNull(); + String content = IoUtils.toUtf8String(result); + assertThat(content).isEqualTo(TEST_CONTENT); + + SdkHttpRequest lastRequest = mockHttpClient.getLastRequest(); + assertThat(lastRequest.method()).isEqualTo(SdkHttpMethod.GET); + assertThat(lastRequest.getUri().toString()).contains("test-bucket.s3.us-east-1.amazonaws.com/test-key"); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + private static HttpExecuteResponse createSuccessResponse() { + SdkHttpFullResponse httpResponse = SdkHttpFullResponse.builder() + .statusCode(200) + .putHeader("Content-Length", "12") + .putHeader("ETag", "\"test-etag\"") + .putHeader("Content-Type", "text/plain") + .build(); + return HttpExecuteResponse.builder() + .response(httpResponse) + .responseBody(AbortableInputStream.create( + new ByteArrayInputStream(TEST_CONTENT.getBytes(StandardCharsets.UTF_8)))) + .build(); + } + + private static HttpExecuteResponse createErrorResponse(int statusCode, String errorCode, String errorMessage) { + String errorContent = String.format( + "%s%s", + errorCode, errorMessage); + SdkHttpFullResponse httpResponse = SdkHttpFullResponse.builder() + .statusCode(statusCode) + .putHeader("x-amz-request-id", "test-request-id") + .putHeader("x-amz-id-2", "test-extended-request-id") + .build(); + return HttpExecuteResponse.builder() + .response(httpResponse) + .responseBody(AbortableInputStream.create( + new ByteArrayInputStream(errorContent.getBytes(StandardCharsets.UTF_8)))) + .build(); + } +} + From ba96aaa0dfa9ec0bee5fda555fa412d141563e8e Mon Sep 17 00:00:00 2001 From: jencymaryjoseph <35571282+jencymaryjoseph@users.noreply.github.com> Date: Wed, 16 Jul 2025 11:52:22 -0700 Subject: [PATCH 04/30] Add AsyncPresignedUrlManager for S3 GetObject operations with presigned URLs (#6255) * async presignedurlmanager api and implementation * Refactor DefaultAsyncPresignedUrlManager: move config to constructor, declare variables inline * change operation name --- .../DefaultAsyncPresignedUrlManager.java | 154 ++++++++ .../AsyncPresignedUrlManager.java | 81 ++++ .../DefaultAsyncPresignedUrlManagerTest.java | 373 ++++++++++++++++++ 3 files changed, 608 insertions(+) create mode 100644 services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/presignedurl/DefaultAsyncPresignedUrlManager.java create mode 100644 services/s3/src/main/java/software/amazon/awssdk/services/s3/presignedurl/AsyncPresignedUrlManager.java create mode 100644 services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/presignedurl/DefaultAsyncPresignedUrlManagerTest.java diff --git a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/presignedurl/DefaultAsyncPresignedUrlManager.java b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/presignedurl/DefaultAsyncPresignedUrlManager.java new file mode 100644 index 000000000000..75c4e581befb --- /dev/null +++ b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/presignedurl/DefaultAsyncPresignedUrlManager.java @@ -0,0 +1,154 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.services.s3.internal.presignedurl; + +import static software.amazon.awssdk.core.client.config.SdkClientOption.SIGNER_OVERRIDDEN; +import static software.amazon.awssdk.utils.FunctionalUtils.runAndLogError; + +import java.util.Collections; +import java.util.List; +import java.util.Optional; +import java.util.concurrent.CompletableFuture; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import software.amazon.awssdk.annotations.SdkInternalApi; +import software.amazon.awssdk.awscore.exception.AwsServiceException; +import software.amazon.awssdk.awscore.internal.AwsProtocolMetadata; +import software.amazon.awssdk.core.async.AsyncResponseTransformer; +import software.amazon.awssdk.core.async.AsyncResponseTransformerUtils; +import software.amazon.awssdk.core.client.config.SdkAdvancedClientOption; +import software.amazon.awssdk.core.client.config.SdkClientConfiguration; +import software.amazon.awssdk.core.client.config.SdkClientOption; +import software.amazon.awssdk.core.client.handler.AsyncClientHandler; +import software.amazon.awssdk.core.client.handler.ClientExecutionParams; +import software.amazon.awssdk.core.exception.SdkClientException; +import software.amazon.awssdk.core.http.HttpResponseHandler; +import software.amazon.awssdk.core.interceptor.SdkInternalExecutionAttribute; +import software.amazon.awssdk.core.metrics.CoreMetric; +import software.amazon.awssdk.core.signer.NoOpSigner; +import software.amazon.awssdk.metrics.MetricCollector; +import software.amazon.awssdk.metrics.MetricPublisher; +import software.amazon.awssdk.metrics.NoOpMetricCollector; +import software.amazon.awssdk.protocols.xml.AwsS3ProtocolFactory; +import software.amazon.awssdk.protocols.xml.XmlOperationMetadata; +import software.amazon.awssdk.services.s3.internal.presignedurl.model.PresignedUrlGetObjectRequestWrapper; +import software.amazon.awssdk.services.s3.model.GetObjectResponse; +import software.amazon.awssdk.services.s3.model.InvalidObjectStateException; +import software.amazon.awssdk.services.s3.model.NoSuchKeyException; +import software.amazon.awssdk.services.s3.model.S3Exception; +import software.amazon.awssdk.services.s3.presignedurl.AsyncPresignedUrlManager; +import software.amazon.awssdk.services.s3.presignedurl.model.PresignedUrlGetObjectRequest; +import software.amazon.awssdk.utils.CompletableFutureUtils; +import software.amazon.awssdk.utils.Pair; + +/** + * Default implementation of {@link AsyncPresignedUrlManager} for executing S3 operations asynchronously using presigned URLs. + */ +@SdkInternalApi +public final class DefaultAsyncPresignedUrlManager implements AsyncPresignedUrlManager { + private static final Logger log = LoggerFactory.getLogger(DefaultAsyncPresignedUrlManager.class); + + private final AsyncClientHandler clientHandler; + private final AwsS3ProtocolFactory protocolFactory; + private final SdkClientConfiguration clientConfiguration; + private final List metricPublishers; + private final AwsProtocolMetadata protocolMetadata; + + public DefaultAsyncPresignedUrlManager(AsyncClientHandler clientHandler, + AwsS3ProtocolFactory protocolFactory, + SdkClientConfiguration clientConfiguration, + AwsProtocolMetadata protocolMetadata) { + this.clientHandler = clientHandler; + this.protocolFactory = protocolFactory; + this.protocolMetadata = protocolMetadata; + this.clientConfiguration = updateSdkClientConfiguration(clientConfiguration); + this.metricPublishers = Optional.ofNullable( + this.clientConfiguration.option(SdkClientOption.METRIC_PUBLISHERS)) + .orElse(Collections.emptyList()); + } + + @Override + public CompletableFuture getObject( + PresignedUrlGetObjectRequest presignedUrlGetObjectRequest, + AsyncResponseTransformer asyncResponseTransformer) + throws NoSuchKeyException, InvalidObjectStateException, + AwsServiceException, SdkClientException, S3Exception { + + PresignedUrlGetObjectRequestWrapper internalRequest = PresignedUrlGetObjectRequestWrapper.builder() + .url(presignedUrlGetObjectRequest.presignedUrl()) + .range(presignedUrlGetObjectRequest.range()) + .build(); + + MetricCollector apiCallMetricCollector = metricPublishers.isEmpty() ? + NoOpMetricCollector.create() : MetricCollector.create("ApiCall"); + + try { + apiCallMetricCollector.reportMetric(CoreMetric.SERVICE_ID, "S3"); + //TODO: Discuss if we need to change OPERATION_NAME as part of Surface API Review + apiCallMetricCollector.reportMetric(CoreMetric.OPERATION_NAME, "GetObject"); + + Pair, CompletableFuture> pair = + AsyncResponseTransformerUtils.wrapWithEndOfStreamFuture(asyncResponseTransformer); + AsyncResponseTransformer finalAsyncResponseTransformer = pair.left(); + asyncResponseTransformer = finalAsyncResponseTransformer; + CompletableFuture endOfStreamFuture = pair.right(); + + HttpResponseHandler responseHandler = protocolFactory.createResponseHandler( + GetObjectResponse::builder, new XmlOperationMetadata().withHasStreamingSuccessResponse(true)); + + HttpResponseHandler errorResponseHandler = protocolFactory.createErrorResponseHandler(); + + CompletableFuture executeFuture = clientHandler.execute( + new ClientExecutionParams() + .withOperationName("GetObject") + .withProtocolMetadata(protocolMetadata) + .withResponseHandler(responseHandler) + .withErrorResponseHandler(errorResponseHandler) + .withRequestConfiguration(clientConfiguration) + .withInput(internalRequest) + .withMetricCollector(apiCallMetricCollector) + // TODO: Deprecate IS_DISCOVERED_ENDPOINT, use new SKIP_ENDPOINT_RESOLUTION for better semantics + .putExecutionAttribute(SdkInternalExecutionAttribute.IS_DISCOVERED_ENDPOINT, true) + .withMarshaller(new PresignedUrlGetObjectRequestMarshaller(protocolFactory)), + asyncResponseTransformer); + + CompletableFuture whenCompleteFuture = executeFuture.whenComplete((r, e) -> { + if (e != null) { + runAndLogError(log, "Exception thrown in exceptionOccurred callback, ignoring", + () -> finalAsyncResponseTransformer.exceptionOccurred(e)); + } + endOfStreamFuture.whenComplete((r2, e2) -> { + metricPublishers.forEach(p -> p.publish(apiCallMetricCollector.collect())); + }); + }); + return CompletableFutureUtils.forwardExceptionTo(whenCompleteFuture, executeFuture); + } catch (Throwable t) { + AsyncResponseTransformer finalAsyncResponseTransformer = asyncResponseTransformer; + runAndLogError(log, "Exception thrown in exceptionOccurred callback, ignoring", + () -> finalAsyncResponseTransformer.exceptionOccurred(t)); + metricPublishers.forEach(p -> p.publish(apiCallMetricCollector.collect())); + return CompletableFutureUtils.failedFuture(t); + } + } + + private SdkClientConfiguration updateSdkClientConfiguration(SdkClientConfiguration clientConfiguration) { + SdkClientConfiguration.Builder configBuilder = clientConfiguration.toBuilder(); + configBuilder.option(SdkAdvancedClientOption.SIGNER, new NoOpSigner()); + configBuilder.option(SIGNER_OVERRIDDEN, true); + return configBuilder.build(); + } + +} diff --git a/services/s3/src/main/java/software/amazon/awssdk/services/s3/presignedurl/AsyncPresignedUrlManager.java b/services/s3/src/main/java/software/amazon/awssdk/services/s3/presignedurl/AsyncPresignedUrlManager.java new file mode 100644 index 000000000000..6342213a679a --- /dev/null +++ b/services/s3/src/main/java/software/amazon/awssdk/services/s3/presignedurl/AsyncPresignedUrlManager.java @@ -0,0 +1,81 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.services.s3.presignedurl; + +import java.util.concurrent.CompletableFuture; +import software.amazon.awssdk.annotations.SdkPublicApi; +import software.amazon.awssdk.core.async.AsyncResponseTransformer; +import software.amazon.awssdk.core.exception.SdkClientException; +import software.amazon.awssdk.services.s3.S3AsyncClient; +import software.amazon.awssdk.services.s3.model.GetObjectResponse; +import software.amazon.awssdk.services.s3.model.InvalidObjectStateException; +import software.amazon.awssdk.services.s3.model.NoSuchKeyException; +import software.amazon.awssdk.services.s3.model.S3Exception; +import software.amazon.awssdk.services.s3.presignedurl.model.PresignedUrlGetObjectRequest; + +/** + * Interface for executing S3 operations asynchronously using presigned URLs. This can be accessed using + * {@link S3AsyncClient#presignedUrlManager()}. + */ +@SdkPublicApi +public interface AsyncPresignedUrlManager { + /** + *

+ * Downloads an S3 object asynchronously using a presigned URL. + *

+ *

+ * This operation uses a presigned URL that contains all necessary authentication information, eliminating the + * need for AWS credentials at request time. The presigned URL must be valid and not expired. + *

+ *
+ *
Range Requests
+ *
+ *

+ * Supports partial object downloads using HTTP Range headers. Specify the range parameter + * in the request to download only a portion of the object (e.g., "bytes=0-1023"). + *

+ *
+ *
+ * + * @param request The presigned URL request containing the URL and optional range parameters + * @param responseTransformer Transforms the response to the desired return type. See + * {@link software.amazon.awssdk.core.async.AsyncResponseTransformer} for pre-built + * implementations like downloading to a file or converting to bytes. + * @param The type of the transformed response + * @return A {@link CompletableFuture} containing the transformed result of the AsyncResponseTransformer + * @throws software.amazon.awssdk.services.s3.model.NoSuchKeyException The specified object does not exist + * @throws software.amazon.awssdk.services.s3.model.InvalidObjectStateException Object is archived and must be restored before + * retrieval + * @throws software.amazon.awssdk.core.exception.SdkClientException If any client side error occurs such as + * network failures or invalid presigned URL + * @throws S3Exception Base class for all S3 service exceptions. + * Unknown exceptions will be thrown as an + * instance of this type. + */ + default CompletableFuture getObject(PresignedUrlGetObjectRequest request, + AsyncResponseTransformer responseTransformer) throws NoSuchKeyException, + InvalidObjectStateException, + SdkClientException, + S3Exception { + throw new UnsupportedOperationException(); + } + + // TODO: Add convenience methods : + // - getObject(Consumer, AsyncResponseTransformer) - consumer-based request building + // - getObject(PresignedUrlGetObjectRequest, Path) - direct file download + // - getObject(Consumer, Path) - consumer + file download +} diff --git a/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/presignedurl/DefaultAsyncPresignedUrlManagerTest.java b/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/presignedurl/DefaultAsyncPresignedUrlManagerTest.java new file mode 100644 index 000000000000..b74ba0089189 --- /dev/null +++ b/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/presignedurl/DefaultAsyncPresignedUrlManagerTest.java @@ -0,0 +1,373 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.services.s3.internal.presignedurl; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.Mockito.any; +import static org.mockito.Mockito.atLeastOnce; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; + +import java.io.ByteArrayInputStream; +import java.net.URI; +import java.net.URL; +import java.nio.charset.StandardCharsets; +import java.util.Collections; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Executors; +import java.util.function.Consumer; +import java.util.stream.Stream; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import org.mockito.ArgumentCaptor; +import software.amazon.awssdk.auth.credentials.AnonymousCredentialsProvider; +import software.amazon.awssdk.awscore.client.config.AwsAdvancedClientOption; +import software.amazon.awssdk.awscore.client.config.AwsClientOption; +import software.amazon.awssdk.awscore.client.handler.AwsAsyncClientHandler; +import software.amazon.awssdk.awscore.internal.AwsProtocolMetadata; +import software.amazon.awssdk.awscore.internal.AwsServiceProtocol; +import software.amazon.awssdk.awscore.retry.AwsRetryStrategy; +import software.amazon.awssdk.core.ResponseBytes; +import software.amazon.awssdk.core.async.AsyncResponseTransformer; +import software.amazon.awssdk.core.client.config.SdkAdvancedAsyncClientOption; +import software.amazon.awssdk.core.client.config.SdkAdvancedClientOption; +import software.amazon.awssdk.core.client.config.SdkClientConfiguration; +import software.amazon.awssdk.core.client.config.SdkClientOption; +import software.amazon.awssdk.core.client.handler.AsyncClientHandler; +import software.amazon.awssdk.core.exception.SdkClientException; +import software.amazon.awssdk.core.metrics.CoreMetric; +import software.amazon.awssdk.http.AbortableInputStream; +import software.amazon.awssdk.http.HttpExecuteResponse; +import software.amazon.awssdk.http.SdkHttpFullResponse; +import software.amazon.awssdk.http.SdkHttpMethod; +import software.amazon.awssdk.http.SdkHttpRequest; +import software.amazon.awssdk.metrics.MetricCollection; +import software.amazon.awssdk.metrics.MetricPublisher; +import software.amazon.awssdk.protocols.core.ExceptionMetadata; +import software.amazon.awssdk.protocols.xml.AwsS3ProtocolFactory; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.s3.model.GetObjectResponse; +import software.amazon.awssdk.services.s3.model.InvalidObjectStateException; +import software.amazon.awssdk.services.s3.model.NoSuchKeyException; +import software.amazon.awssdk.services.s3.model.S3Exception; +import software.amazon.awssdk.services.s3.presignedurl.model.PresignedUrlGetObjectRequest; +import software.amazon.awssdk.testutils.service.http.MockAsyncHttpClient; + +/** + * Tests for {@link DefaultAsyncPresignedUrlManager} using MockAsyncHttpClient to verify HTTP interactions. + */ +class DefaultAsyncPresignedUrlManagerTest { + + private static final String TEST_CONTENT = "test-content"; + private static final URI DEFAULT_ENDPOINT = URI.create("https://defaultendpoint.com"); + private static final String TEST_URL = "https://test-bucket.s3.us-east-1.amazonaws.com/test-key?" + + "X-Amz-Date=20250707T000000Z&" + + "X-Amz-Signature=test-signature-value&" + + "X-Amz-Algorithm=AWS4-HMAC-SHA256&" + + "X-Amz-SignedHeaders=host&" + + "X-Amz-Security-Token=test-session-token&" + + "X-Amz-Credential=AKIAIOSFODNN7EXAMPLE%2F20250707%2Fus-east-1%2Fs3%2Faws4_request&" + + "X-Amz-Expires=86400"; + + private MockAsyncHttpClient mockHttpClient; + private DefaultAsyncPresignedUrlManager presignedUrlManager; + private PresignedUrlGetObjectRequest testRequest; + private AwsProtocolMetadata protocolMetadata; + private AwsS3ProtocolFactory protocolFactory; + private AsyncClientHandler clientHandler; + + @BeforeEach + void setUp() throws Exception { + mockHttpClient = new MockAsyncHttpClient(); + URL testPresignedUrl = new URL(TEST_URL); + testRequest = PresignedUrlGetObjectRequest.builder() + .presignedUrl(testPresignedUrl) + .build(); + + SdkClientConfiguration clientConfiguration = getDefaultSdkConfigs(); + protocolMetadata = AwsProtocolMetadata.builder() + .serviceProtocol(AwsServiceProtocol.REST_XML) + .build(); + protocolFactory = initProtocolFactory(clientConfiguration); + clientHandler = new AwsAsyncClientHandler(clientConfiguration); + + presignedUrlManager = new DefaultAsyncPresignedUrlManager( + clientHandler, protocolFactory, clientConfiguration, protocolMetadata); + } + + private static Stream httpResponseTestCases() { + return Stream.of( + Arguments.of( + "Success response", + createSuccessResponse(), + true, + null + ), + Arguments.of( + "404 Not Found response", + createErrorResponse(404, "NoSuchKey", "The specified key does not exist."), + false, + NoSuchKeyException.class + ), + Arguments.of( + "403 Invalid Object State response", + createErrorResponse(403, "InvalidObjectState", "The operation is not valid for the object's storage class."), + false, + InvalidObjectStateException.class + ), + Arguments.of( + "Generic error response", + createErrorResponse(500, "InternalError", "We encountered an internal error. Please try again."), + false, + S3Exception.class + ) + ); + } + + private static Stream requestConfigurationTestCases() { + return Stream.of( + Arguments.of( + "Basic request", + (Consumer) builder -> + builder.presignedUrl(createTestUrl()), + null + ), + Arguments.of( + "Request with range header", + (Consumer) builder -> + builder.presignedUrl(createTestUrl()).range("bytes=0-1024"), + "bytes=0-1024" + ) + ); + } + + private static Stream additionalTestCases() { + return Stream.of( + Arguments.of("Custom transformer test", "CUSTOM_TRANSFORMER"), + Arguments.of("Metrics collection test", "METRICS_COLLECTION") + ); + } + + private static Stream invalidUrlTestCases() { + return Stream.of( + Arguments.of("Invalid URL format", "not-a-url"), + Arguments.of("Empty HTTP URL", "http://"), + Arguments.of("Empty HTTPS URL", "https://"), + Arguments.of("Empty string", "") + ); + } + + @ParameterizedTest(name = "{0}") + @MethodSource("httpResponseTestCases") + void given_AsyncPresignedUrlManager_when_GetObjectWithDifferentHttpResponses_then_ShouldHandleSuccessAndErrorsCorrectly( + String testName, + HttpExecuteResponse response, + boolean expectSuccess, + Class expectedExceptionType) { + + mockHttpClient.stubNextResponse(response); + + if (expectSuccess) { + assertSuccessfulGetObject(testRequest); + } else { + CompletableFuture> future = + presignedUrlManager.getObject(testRequest, AsyncResponseTransformer.toBytes()); + + assertThatThrownBy(future::join) + .hasCauseInstanceOf(expectedExceptionType); + } + } + + @ParameterizedTest(name = "{0}") + @MethodSource("requestConfigurationTestCases") + void given_AsyncPresignedUrlManager_when_GetObjectWithDifferentRequestConfigurations_then_ShouldSetCorrectHeaders( + String testName, + Consumer requestCustomizer, + String expectedRangeHeader) throws ExecutionException, InterruptedException { + + mockHttpClient.stubNextResponse(createSuccessResponse()); + + PresignedUrlGetObjectRequest.Builder builder = PresignedUrlGetObjectRequest.builder(); + requestCustomizer.accept(builder); + PresignedUrlGetObjectRequest request = builder.build(); + + CompletableFuture> future = + presignedUrlManager.getObject(request, AsyncResponseTransformer.toBytes()); + ResponseBytes result = future.get(); + + assertThat(result).isNotNull(); + assertThat(result.asUtf8String()).isEqualTo(TEST_CONTENT); + + SdkHttpRequest lastRequest = mockHttpClient.getLastRequest(); + assertThat(lastRequest.method()).isEqualTo(SdkHttpMethod.GET); + + if (expectedRangeHeader != null) { + assertThat(lastRequest.firstMatchingHeader("Range")) + .isPresent() + .contains(expectedRangeHeader); + } + } + + @ParameterizedTest(name = "{0}") + @MethodSource("additionalTestCases") + void given_AsyncPresignedUrlManager_when_ExecutingDifferentScenarios_then_ShouldBehaveCorrectly( + String testName, String testType) throws Exception { + + switch (testType) { + case "CUSTOM_TRANSFORMER": + mockHttpClient.stubNextResponse(createSuccessResponse()); + CompletableFuture> future = + presignedUrlManager.getObject(testRequest, AsyncResponseTransformer.toBytes()); + ResponseBytes result = future.get(); + assertThat(result.asUtf8String()).isEqualTo(TEST_CONTENT); + break; + + case "METRICS_COLLECTION": + MetricPublisher mockPublisher = mock(MetricPublisher.class); + SdkClientConfiguration clientConfigWithMetrics = getDefaultSdkConfigs().toBuilder() + .option(SdkClientOption.METRIC_PUBLISHERS, Collections.singletonList(mockPublisher)) + .build(); + DefaultAsyncPresignedUrlManager managerWithMetrics = new DefaultAsyncPresignedUrlManager( + clientHandler, protocolFactory, clientConfigWithMetrics, protocolMetadata); + mockHttpClient.stubNextResponse(createSuccessResponse()); + CompletableFuture> metricsFuture = + managerWithMetrics.getObject(testRequest, AsyncResponseTransformer.toBytes()); + metricsFuture.get(); + + verify(mockPublisher, atLeastOnce()).publish(any(MetricCollection.class)); + ArgumentCaptor metricsCaptor = ArgumentCaptor.forClass(MetricCollection.class); + verify(mockPublisher).publish(metricsCaptor.capture()); + MetricCollection capturedMetrics = metricsCaptor.getValue(); + assertThat(capturedMetrics.metricValues(CoreMetric.SERVICE_ID)).contains("S3"); + assertThat(capturedMetrics.metricValues(CoreMetric.OPERATION_NAME)).contains("GetObject"); + break; + } + } + + @ParameterizedTest(name = "{0}") + @MethodSource("invalidUrlTestCases") + void given_AsyncPresignedUrlManager_when_GetObjectWithInvalidUrl_then_ShouldThrowException( + String testName, String invalidUrlString) { + assertThatThrownBy(() -> { + URL invalidUrl = new URL(invalidUrlString); + PresignedUrlGetObjectRequest invalidRequest = PresignedUrlGetObjectRequest.builder() + .presignedUrl(invalidUrl) + .build(); + presignedUrlManager.getObject(invalidRequest, AsyncResponseTransformer.toBytes()).join(); + }).satisfiesAnyOf( + ex -> assertThat(ex).isInstanceOf(java.net.MalformedURLException.class), + ex -> assertThat(ex).isInstanceOf(SdkClientException.class), + ex -> assertThat(ex).isInstanceOf(java.util.concurrent.CompletionException.class) + .extracting(Throwable::getCause) + .isInstanceOf(SdkClientException.class) + ); + } + + private SdkClientConfiguration getDefaultSdkConfigs() { + return SdkClientConfiguration.builder() + .option(SdkClientOption.ASYNC_HTTP_CLIENT, mockHttpClient) + .option(SdkClientOption.ADDITIONAL_HTTP_HEADERS, Collections.emptyMap()) + .option(SdkClientOption.EXECUTION_INTERCEPTORS, Collections.emptyList()) + .option(SdkClientOption.RETRY_STRATEGY, AwsRetryStrategy.doNotRetry()) + .option(SdkAdvancedClientOption.USER_AGENT_PREFIX, "") + .option(SdkAdvancedClientOption.USER_AGENT_SUFFIX, "") + .option(SdkClientOption.CRC32_FROM_COMPRESSED_DATA_ENABLED, false) + .option(AwsClientOption.CREDENTIALS_IDENTITY_PROVIDER, AnonymousCredentialsProvider.create()) + .option(AwsClientOption.AWS_REGION, Region.US_EAST_2) + .option(AwsClientOption.SIGNING_REGION, Region.US_EAST_2) + .option(AwsClientOption.SERVICE_SIGNING_NAME, Region.AP_EAST_2.toString()) + .option(AwsAdvancedClientOption.ENABLE_DEFAULT_REGION_DETECTION, false) + .option(SdkClientOption.SCHEDULED_EXECUTOR_SERVICE, Executors.newScheduledThreadPool(1)) + .option(SdkAdvancedAsyncClientOption.FUTURE_COMPLETION_EXECUTOR, Runnable::run) + .option(SdkClientOption.CLIENT_ENDPOINT_PROVIDER, + software.amazon.awssdk.core.ClientEndpointProvider.forEndpointOverride(DEFAULT_ENDPOINT)) + .build(); + } + + private AwsS3ProtocolFactory initProtocolFactory(SdkClientConfiguration configuration) { + return AwsS3ProtocolFactory.builder() + .registerModeledException( + ExceptionMetadata.builder().errorCode("NoSuchKey") + .exceptionBuilderSupplier(NoSuchKeyException::builder) + .httpStatusCode(404).build()) + .registerModeledException( + ExceptionMetadata.builder().errorCode("InvalidObjectState") + .exceptionBuilderSupplier(InvalidObjectStateException::builder) + .httpStatusCode(403).build()) + .clientConfiguration(configuration) + .defaultServiceExceptionSupplier(S3Exception::builder) + .build(); + } + + private static URL createTestUrl() { + try { + return new URL(TEST_URL); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + private void assertSuccessfulGetObject(PresignedUrlGetObjectRequest request) { + try { + CompletableFuture> future = + presignedUrlManager.getObject(request, AsyncResponseTransformer.toBytes()); + ResponseBytes result = future.get(); + + assertThat(result).isNotNull(); + assertThat(result.asUtf8String()).isEqualTo(TEST_CONTENT); + + SdkHttpRequest lastRequest = mockHttpClient.getLastRequest(); + assertThat(lastRequest.method()).isEqualTo(SdkHttpMethod.GET); + assertThat(lastRequest.getUri().toString()).contains("test-bucket.s3.us-east-1.amazonaws.com/test-key"); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + private static HttpExecuteResponse createSuccessResponse() { + SdkHttpFullResponse httpResponse = SdkHttpFullResponse.builder() + .statusCode(200) + .putHeader("Content-Length", "12") + .putHeader("ETag", "\"test-etag\"") + .putHeader("Content-Type", "text/plain") + .build(); + return HttpExecuteResponse.builder() + .response(httpResponse) + .responseBody(AbortableInputStream.create( + new ByteArrayInputStream(TEST_CONTENT.getBytes(StandardCharsets.UTF_8)))) + .build(); + } + + private static HttpExecuteResponse createErrorResponse(int statusCode, String errorCode, String errorMessage) { + String errorContent = String.format( + "%s%s", + errorCode, errorMessage); + SdkHttpFullResponse httpResponse = SdkHttpFullResponse.builder() + .statusCode(statusCode) + .putHeader("x-amz-request-id", "test-request-id") + .putHeader("x-amz-id-2", "test-extended-request-id") + .build(); + return HttpExecuteResponse.builder() + .response(httpResponse) + .responseBody(AbortableInputStream.create( + new ByteArrayInputStream(errorContent.getBytes(StandardCharsets.UTF_8)))) + .build(); + } +} \ No newline at end of file From 9baca167b389278e06a1ef5c824073006991aec2 Mon Sep 17 00:00:00 2001 From: jencymaryjoseph <35571282+jencymaryjoseph@users.noreply.github.com> Date: Thu, 17 Jul 2025 09:39:05 -0700 Subject: [PATCH 05/30] Removed presignedurl api and implementation for sync client (#6270) --- .../DefaultPresignedUrlManager.java | 122 ------ .../s3/presignedurl/PresignedUrlManager.java | 84 ---- .../DefaultPresignedUrlManagerTest.java | 378 ------------------ 3 files changed, 584 deletions(-) delete mode 100644 services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/presignedurl/DefaultPresignedUrlManager.java delete mode 100644 services/s3/src/main/java/software/amazon/awssdk/services/s3/presignedurl/PresignedUrlManager.java delete mode 100644 services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/presignedurl/DefaultPresignedUrlManagerTest.java diff --git a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/presignedurl/DefaultPresignedUrlManager.java b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/presignedurl/DefaultPresignedUrlManager.java deleted file mode 100644 index 3527b4d830d4..000000000000 --- a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/presignedurl/DefaultPresignedUrlManager.java +++ /dev/null @@ -1,122 +0,0 @@ -/* - * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://aws.amazon.com/apache2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package software.amazon.awssdk.services.s3.internal.presignedurl; - -import static software.amazon.awssdk.core.client.config.SdkClientOption.SIGNER_OVERRIDDEN; - -import java.util.Collections; -import java.util.List; -import java.util.Optional; -import software.amazon.awssdk.annotations.SdkInternalApi; -import software.amazon.awssdk.awscore.exception.AwsServiceException; -import software.amazon.awssdk.awscore.internal.AwsProtocolMetadata; -import software.amazon.awssdk.core.client.config.SdkAdvancedClientOption; -import software.amazon.awssdk.core.client.config.SdkClientConfiguration; -import software.amazon.awssdk.core.client.config.SdkClientOption; -import software.amazon.awssdk.core.client.handler.ClientExecutionParams; -import software.amazon.awssdk.core.client.handler.SyncClientHandler; -import software.amazon.awssdk.core.exception.SdkClientException; -import software.amazon.awssdk.core.http.HttpResponseHandler; -import software.amazon.awssdk.core.interceptor.SdkInternalExecutionAttribute; -import software.amazon.awssdk.core.metrics.CoreMetric; -import software.amazon.awssdk.core.signer.NoOpSigner; -import software.amazon.awssdk.core.sync.ResponseTransformer; -import software.amazon.awssdk.metrics.MetricCollector; -import software.amazon.awssdk.metrics.MetricPublisher; -import software.amazon.awssdk.metrics.NoOpMetricCollector; -import software.amazon.awssdk.protocols.xml.AwsS3ProtocolFactory; -import software.amazon.awssdk.protocols.xml.XmlOperationMetadata; -import software.amazon.awssdk.services.s3.internal.presignedurl.model.PresignedUrlGetObjectRequestWrapper; -import software.amazon.awssdk.services.s3.model.GetObjectResponse; -import software.amazon.awssdk.services.s3.model.InvalidObjectStateException; -import software.amazon.awssdk.services.s3.model.NoSuchKeyException; -import software.amazon.awssdk.services.s3.model.S3Exception; -import software.amazon.awssdk.services.s3.presignedurl.PresignedUrlManager; -import software.amazon.awssdk.services.s3.presignedurl.model.PresignedUrlGetObjectRequest; - -/** - * Default implementation of {@link PresignedUrlManager} for executing S3 operations using presigned URLs. - */ -@SdkInternalApi -public final class DefaultPresignedUrlManager implements PresignedUrlManager { - - private final SyncClientHandler clientHandler; - private final AwsS3ProtocolFactory protocolFactory; - private final SdkClientConfiguration clientConfiguration; - private final AwsProtocolMetadata protocolMetadata; - - public DefaultPresignedUrlManager(SyncClientHandler clientHandler, - AwsS3ProtocolFactory protocolFactory, - SdkClientConfiguration clientConfiguration, - AwsProtocolMetadata protocolMetadata) { - this.clientHandler = clientHandler; - this.protocolFactory = protocolFactory; - this.clientConfiguration = clientConfiguration; - this.protocolMetadata = protocolMetadata; - } - - @Override - public ReturnT getObject(PresignedUrlGetObjectRequest presignedUrlGetObjectRequest, - ResponseTransformer responseTransformer) - throws NoSuchKeyException, InvalidObjectStateException, - AwsServiceException, SdkClientException, S3Exception { - - HttpResponseHandler responseHandler = protocolFactory.createResponseHandler( - GetObjectResponse::builder, new XmlOperationMetadata().withHasStreamingSuccessResponse(true)); - - HttpResponseHandler errorResponseHandler = protocolFactory.createErrorResponseHandler(); - - PresignedUrlGetObjectRequestWrapper internalRequest = PresignedUrlGetObjectRequestWrapper.builder() - .url(presignedUrlGetObjectRequest.presignedUrl()) - .range(presignedUrlGetObjectRequest.range()) - .build(); - - SdkClientConfiguration updatedClientConfiguration = updateSdkClientConfiguration(this.clientConfiguration); - List metricPublishers = Optional.ofNullable( - updatedClientConfiguration.option(SdkClientOption.METRIC_PUBLISHERS)) - .orElse(Collections.emptyList()); - MetricCollector apiCallMetricCollector = metricPublishers.isEmpty() ? - NoOpMetricCollector.create() : MetricCollector.create("ApiCall"); - try { - apiCallMetricCollector.reportMetric(CoreMetric.SERVICE_ID, "S3"); - //TODO: Discuss if we need to change OPERATION_NAME as part of Surface API Review - apiCallMetricCollector.reportMetric(CoreMetric.OPERATION_NAME, "GetObject"); - - return clientHandler.execute( - new ClientExecutionParams() - .withOperationName("PresignedUrlGetObject") - .withProtocolMetadata(protocolMetadata) - .withResponseHandler(responseHandler) - .withErrorResponseHandler(errorResponseHandler) - .withRequestConfiguration(updatedClientConfiguration) - .withInput(internalRequest) - .withMetricCollector(apiCallMetricCollector) - // TODO: Deprecate IS_DISCOVERED_ENDPOINT, use new SKIP_ENDPOINT_RESOLUTION for better semantics - .putExecutionAttribute(SdkInternalExecutionAttribute.IS_DISCOVERED_ENDPOINT, true) - .withMarshaller(new PresignedUrlGetObjectRequestMarshaller(protocolFactory)), responseTransformer); - } finally { - metricPublishers.forEach(p -> p.publish(apiCallMetricCollector.collect())); - } - } - - private SdkClientConfiguration updateSdkClientConfiguration(SdkClientConfiguration clientConfiguration) { - SdkClientConfiguration.Builder configBuilder = clientConfiguration.toBuilder(); - configBuilder.option(SdkAdvancedClientOption.SIGNER, new NoOpSigner()); - configBuilder.option(SIGNER_OVERRIDDEN, true); - return configBuilder.build(); - } - -} diff --git a/services/s3/src/main/java/software/amazon/awssdk/services/s3/presignedurl/PresignedUrlManager.java b/services/s3/src/main/java/software/amazon/awssdk/services/s3/presignedurl/PresignedUrlManager.java deleted file mode 100644 index 3b005ce871da..000000000000 --- a/services/s3/src/main/java/software/amazon/awssdk/services/s3/presignedurl/PresignedUrlManager.java +++ /dev/null @@ -1,84 +0,0 @@ -/* - * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://aws.amazon.com/apache2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package software.amazon.awssdk.services.s3.presignedurl; - -import software.amazon.awssdk.annotations.SdkPublicApi; -import software.amazon.awssdk.core.exception.SdkClientException; -import software.amazon.awssdk.core.sync.ResponseTransformer; -import software.amazon.awssdk.services.s3.S3Client; -import software.amazon.awssdk.services.s3.model.GetObjectResponse; -import software.amazon.awssdk.services.s3.model.InvalidObjectStateException; -import software.amazon.awssdk.services.s3.model.NoSuchKeyException; -import software.amazon.awssdk.services.s3.model.S3Exception; -import software.amazon.awssdk.services.s3.presignedurl.model.PresignedUrlGetObjectRequest; - -/** - * Interface for executing S3 operations using presigned URLs. This can be accessed using {@link S3Client#presignedUrlManager()}. - */ -@SdkPublicApi -public interface PresignedUrlManager { - /** - *

- * Downloads an S3 object using a presigned URL. - *

- *

- * This operation uses a presigned URL that contains all necessary authentication information, eliminating the need for AWS - * credentials at request time. The presigned URL must be valid and not expired. - *

- *
- *
Range Requests
- *
- *

- * Supports partial object downloads using HTTP Range headers. Specify the range parameter - * in the request to download only a portion of the object (e.g., "bytes=0-1023"). - *

- *
- *
Storage Classes
- *
- *

- * If the object is stored in S3 Glacier Flexible Retrieval, S3 Glacier Deep Archive, - * or S3 Intelligent-Tiering Archive tiers, the object must be restored before retrieval. - * Otherwise, this operation returns an InvalidObjectState error. - *

- *
- *
- * - * @param request The presigned URL request containing the URL and optional range parameters - * @param responseTransformer Transforms the response to the desired return type. See - * {@link software.amazon.awssdk.core.sync.ResponseTransformer} for pre-built implementations like - * downloading to a file or converting to bytes. - * @param The type of the transformed response - * @return The transformed result of the ResponseTransformer - * @throws software.amazon.awssdk.services.s3.model.NoSuchKeyException The specified object does not exist - * @throws software.amazon.awssdk.services.s3.model.InvalidObjectStateException Object is archived and must be restored before - * retrieval - * @throws software.amazon.awssdk.core.exception.SdkClientException If any client side error occurs such as - * network failures or invalid presigned URL - * @throws S3Exception Base class for all S3 service exceptions. - * Unknown exceptions will be thrown as an - * instance of this type. - */ - default ReturnT getObject(PresignedUrlGetObjectRequest request, - ResponseTransformer responseTransformer) throws NoSuchKeyException, - InvalidObjectStateException, - SdkClientException, - S3Exception { - throw new UnsupportedOperationException(); - } - - //TODO: Add other getObject method flavors -} diff --git a/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/presignedurl/DefaultPresignedUrlManagerTest.java b/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/presignedurl/DefaultPresignedUrlManagerTest.java deleted file mode 100644 index ff6266689927..000000000000 --- a/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/presignedurl/DefaultPresignedUrlManagerTest.java +++ /dev/null @@ -1,378 +0,0 @@ -/* - * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://aws.amazon.com/apache2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package software.amazon.awssdk.services.s3.internal.presignedurl; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.assertThatThrownBy; -import java.io.ByteArrayInputStream; -import java.io.IOException; -import java.net.URI; -import java.net.URL; -import java.nio.charset.StandardCharsets; -import java.util.Collections; -import java.util.function.Consumer; -import java.util.stream.Stream; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.Arguments; -import org.junit.jupiter.params.provider.MethodSource; -import org.mockito.ArgumentCaptor; -import software.amazon.awssdk.awscore.retry.AwsRetryStrategy; -import software.amazon.awssdk.core.exception.SdkClientException; -import software.amazon.awssdk.metrics.MetricCollection; -import software.amazon.awssdk.metrics.MetricPublisher; -import software.amazon.awssdk.core.metrics.CoreMetric; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.atLeastOnce; -import static org.mockito.Mockito.any; -import software.amazon.awssdk.auth.credentials.AnonymousCredentialsProvider; -import software.amazon.awssdk.awscore.client.config.AwsAdvancedClientOption; -import software.amazon.awssdk.awscore.client.config.AwsClientOption; -import software.amazon.awssdk.awscore.client.handler.AwsSyncClientHandler; -import software.amazon.awssdk.awscore.internal.AwsProtocolMetadata; -import software.amazon.awssdk.awscore.internal.AwsServiceProtocol; -import software.amazon.awssdk.core.ResponseInputStream; -import software.amazon.awssdk.core.client.config.SdkAdvancedClientOption; -import software.amazon.awssdk.core.client.config.SdkClientConfiguration; -import software.amazon.awssdk.core.client.config.SdkClientOption; -import software.amazon.awssdk.core.client.handler.SyncClientHandler; -import software.amazon.awssdk.core.sync.ResponseTransformer; -import software.amazon.awssdk.http.AbortableInputStream; -import software.amazon.awssdk.http.HttpExecuteResponse; -import software.amazon.awssdk.http.SdkHttpFullResponse; -import software.amazon.awssdk.http.SdkHttpMethod; -import software.amazon.awssdk.http.SdkHttpRequest; -import software.amazon.awssdk.protocols.core.ExceptionMetadata; -import software.amazon.awssdk.protocols.xml.AwsS3ProtocolFactory; -import software.amazon.awssdk.regions.Region; -import software.amazon.awssdk.services.s3.model.GetObjectResponse; -import software.amazon.awssdk.services.s3.model.InvalidObjectStateException; -import software.amazon.awssdk.services.s3.model.NoSuchKeyException; -import software.amazon.awssdk.services.s3.model.S3Exception; -import software.amazon.awssdk.services.s3.presignedurl.model.PresignedUrlGetObjectRequest; -import software.amazon.awssdk.testutils.service.http.MockSyncHttpClient; -import software.amazon.awssdk.utils.IoUtils; - -/** - * Tests for {@link DefaultPresignedUrlManager} using MockSyncHttpClient to verify HTTP interactions. - */ - class DefaultPresignedUrlManagerTest { - - private static final String TEST_CONTENT = "test-content"; - private static final URI DEFAULT_ENDPOINT = URI.create("https://defaultendpoint.com"); - private static final String TEST_URL = "https://test-bucket.s3.us-east-1.amazonaws.com/test-key?" + - "X-Amz-Date=20250707T000000Z&" + - "X-Amz-Signature=test-signature-value&" + - "X-Amz-Algorithm=AWS4-HMAC-SHA256&" + - "X-Amz-SignedHeaders=host&" + - "X-Amz-Security-Token=test-session-token&" + - "X-Amz-Credential=AKIAIOSFODNN7EXAMPLE%2F20250707%2Fus-east-1%2Fs3%2Faws4_request&" + - "X-Amz-Expires=86400"; - private MockSyncHttpClient mockHttpClient; - private DefaultPresignedUrlManager presignedUrlManager; - private URL testPresignedUrl; - private PresignedUrlGetObjectRequest testRequest; - private AwsProtocolMetadata protocolMetadata; - private AwsS3ProtocolFactory protocolFactory; - private SyncClientHandler clientHandler; - private SdkClientConfiguration clientConfiguration; - - @BeforeEach - void setUp() throws Exception { - mockHttpClient = new MockSyncHttpClient(); - testPresignedUrl = new URL(TEST_URL); - testRequest = PresignedUrlGetObjectRequest.builder() - .presignedUrl(testPresignedUrl) - .build(); - - clientConfiguration = getDefaultSdkConfigs(); - protocolMetadata = AwsProtocolMetadata.builder() - .serviceProtocol(AwsServiceProtocol.REST_XML) - .build(); - protocolFactory = initProtocolFactory(clientConfiguration); - clientHandler = new AwsSyncClientHandler(clientConfiguration); - - presignedUrlManager = new DefaultPresignedUrlManager( - clientHandler, protocolFactory, clientConfiguration, protocolMetadata); - } - - private static Stream httpResponseTestCases() { - return Stream.of( - Arguments.of( - "Success response", - createSuccessResponse(), - true, - null - ), - Arguments.of( - "404 Not Found response", - createErrorResponse(404, "NoSuchKey", "The specified key does not exist."), - false, - NoSuchKeyException.class - ), - Arguments.of( - "403 Invalid Object State response", - createErrorResponse(403, "InvalidObjectState", "The operation is not valid for the object's storage class."), - false, - InvalidObjectStateException.class - ), - Arguments.of( - "Generic error response", - createErrorResponse(500, "InternalError", "We encountered an internal error. Please try again."), - false, - S3Exception.class - ) - ); - } - - private static Stream requestConfigurationTestCases() { - return Stream.of( - Arguments.of( - "Basic request", - (Consumer) builder -> - builder.presignedUrl(createTestUrl()), - null - ), - Arguments.of( - "Request with range header", - (Consumer) builder -> - builder.presignedUrl(createTestUrl()).range("bytes=0-1024"), - "bytes=0-1024" - ) - ); - } - - @ParameterizedTest(name = "{0}") - @MethodSource("httpResponseTestCases") - void given_PresignedUrlManager_when_GetObjectWithDifferentHttpResponses_then_ShouldHandleSuccessAndErrorsCorrectly(String testName, - HttpExecuteResponse response, - boolean expectSuccess, - Class expectedExceptionType) { - mockHttpClient.stubNextResponse(response); - if (expectSuccess) { - assertSuccessfulGetObject(testRequest); - } else { - assertThatThrownBy(() -> presignedUrlManager.getObject(testRequest, ResponseTransformer.toInputStream())) - .isInstanceOf(expectedExceptionType); - } - } - - @ParameterizedTest(name = "{0}") - @MethodSource("requestConfigurationTestCases") - void given_PresignedUrlManager_when_GetObjectWithDifferentRequestConfigurations_then_ShouldSetCorrectHeaders(String testName, - Consumer requestCustomizer, - String expectedRangeHeader) throws IOException { - mockHttpClient.stubNextResponse(createSuccessResponse()); - PresignedUrlGetObjectRequest.Builder builder = PresignedUrlGetObjectRequest.builder(); - requestCustomizer.accept(builder); - PresignedUrlGetObjectRequest request = builder.build(); - ResponseInputStream result = presignedUrlManager.getObject(request, ResponseTransformer.toInputStream()); - assertThat(result).isNotNull(); - String content = IoUtils.toUtf8String(result); - assertThat(content).isEqualTo(TEST_CONTENT); - SdkHttpRequest lastRequest = mockHttpClient.getLastRequest(); - assertThat(lastRequest.method()).isEqualTo(SdkHttpMethod.GET); - if (expectedRangeHeader != null) { - assertThat(lastRequest.firstMatchingHeader("Range")).isPresent() - .contains(expectedRangeHeader); - } - } - - private static Stream additionalTestCases() { - return Stream.of( - Arguments.of( - "Custom transformer test", - "CUSTOM_TRANSFORMER" - ), - Arguments.of( - "Resolved endpoint verification", - "ENDPOINT_VERIFICATION" - ), - Arguments.of( - "Metrics collection test", - "METRICS_COLLECTION" - ) - ); - } - - private static Stream invalidUrlTestCases() { - return Stream.of( - Arguments.of( - "URL with spaces in path", - "https://test-bucket.s3.us-east-1.amazonaws.com/test key with spaces" - ), - Arguments.of( - "URL with invalid characters", - "https://test-bucket.s3.us-east-1.amazonaws.com/test<>key" - ), - Arguments.of( - "Malformed URL", - "not-a-valid-url" - ) - ); - } - - @ParameterizedTest(name = "{0}") - @MethodSource("additionalTestCases") - void given_PresignedUrlManager_when_ExecutingDifferentScenarios_then_ShouldBehaveCorrectly(String testName, - String testType) { - switch (testType) { - case "CUSTOM_TRANSFORMER": - mockHttpClient.stubNextResponse(createSuccessResponse()); - ResponseTransformer transformer = - (response, inputStream) -> IoUtils.toUtf8String(inputStream); - String result = presignedUrlManager.getObject(testRequest, transformer); - assertThat(result).isEqualTo(TEST_CONTENT); - break; - - case "ENDPOINT_VERIFICATION": - mockHttpClient.stubNextResponse(createSuccessResponse()); - presignedUrlManager.getObject(testRequest, ResponseTransformer.toInputStream()); - SdkHttpRequest lastRequest = mockHttpClient.getLastRequest(); - assertThat(lastRequest.getUri().toString()).startsWith(testPresignedUrl.toString().split("\\?")[0]); - String presignedUrlQuery = testPresignedUrl.getQuery(); - for (String param : presignedUrlQuery.split("&")) { - String[] keyValue = param.split("=", 2); - if (keyValue.length == 2) { - String key = keyValue[0]; - String value = keyValue[1]; - assertThat(lastRequest.getUri().toString()).contains(key + "=" + value); - } - } - break; - - case "METRICS_COLLECTION": - MetricPublisher mockPublisher = mock(MetricPublisher.class); - SdkClientConfiguration clientConfigWithMetrics = getDefaultSdkConfigs().toBuilder() - .option(SdkClientOption.METRIC_PUBLISHERS, Collections.singletonList(mockPublisher)) - .build(); - DefaultPresignedUrlManager managerWithMetrics = new DefaultPresignedUrlManager( - clientHandler, protocolFactory, clientConfigWithMetrics, protocolMetadata); - mockHttpClient.stubNextResponse(createSuccessResponse()); - managerWithMetrics.getObject(testRequest, ResponseTransformer.toInputStream()); - verify(mockPublisher, atLeastOnce()).publish(any(MetricCollection.class)); - ArgumentCaptor metricsCaptor = ArgumentCaptor.forClass(MetricCollection.class); - verify(mockPublisher).publish(metricsCaptor.capture()); - MetricCollection capturedMetrics = metricsCaptor.getValue(); - assertThat(capturedMetrics.metricValues(CoreMetric.SERVICE_ID)).contains("S3"); - assertThat(capturedMetrics.metricValues(CoreMetric.OPERATION_NAME)).contains("GetObject"); - break; - } - } - - @ParameterizedTest(name = "{0}") - @MethodSource("invalidUrlTestCases") - void given_PresignedUrlManager_when_GetObjectWithInvalidUrl_then_ShouldThrowException(String testName, String invalidUrlString) { - assertThatThrownBy(() -> { - URL invalidUrl = new URL(invalidUrlString); - PresignedUrlGetObjectRequest invalidRequest = PresignedUrlGetObjectRequest.builder() - .presignedUrl(invalidUrl) - .build(); - presignedUrlManager.getObject(invalidRequest, ResponseTransformer.toInputStream()); - }).satisfiesAnyOf( - ex -> assertThat(ex).isInstanceOf(java.net.MalformedURLException.class), - ex -> assertThat(ex).isInstanceOf(SdkClientException.class) - ); - } - - private SdkClientConfiguration getDefaultSdkConfigs() { - return SdkClientConfiguration.builder() - .option(SdkClientOption.SYNC_HTTP_CLIENT, mockHttpClient) - .option(SdkClientOption.ADDITIONAL_HTTP_HEADERS, Collections.emptyMap()) - .option(SdkClientOption.EXECUTION_INTERCEPTORS, Collections.emptyList()) - .option(SdkClientOption.RETRY_STRATEGY, AwsRetryStrategy.doNotRetry()) - .option(SdkAdvancedClientOption.USER_AGENT_PREFIX, "") - .option(SdkAdvancedClientOption.USER_AGENT_SUFFIX, "") - .option(SdkClientOption.CRC32_FROM_COMPRESSED_DATA_ENABLED, false) - .option(AwsClientOption.CREDENTIALS_IDENTITY_PROVIDER, AnonymousCredentialsProvider.create()) - .option(AwsClientOption.AWS_REGION, Region.US_EAST_2) - .option(AwsClientOption.SIGNING_REGION, Region.US_EAST_2) - .option(AwsClientOption.SERVICE_SIGNING_NAME, Region.AP_EAST_2.toString()) - .option(AwsAdvancedClientOption.ENABLE_DEFAULT_REGION_DETECTION, false) - .option(SdkClientOption.CLIENT_ENDPOINT_PROVIDER, - software.amazon.awssdk.core.ClientEndpointProvider.forEndpointOverride(DEFAULT_ENDPOINT)) - .build(); - } - - private AwsS3ProtocolFactory initProtocolFactory(SdkClientConfiguration configuration) { - return AwsS3ProtocolFactory.builder() - .registerModeledException( - ExceptionMetadata.builder().errorCode("NoSuchKey") - .exceptionBuilderSupplier(NoSuchKeyException::builder) - .httpStatusCode(404).build()) - .registerModeledException( - ExceptionMetadata.builder().errorCode("InvalidObjectState") - .exceptionBuilderSupplier(InvalidObjectStateException::builder) - .httpStatusCode(403).build()) - .clientConfiguration(configuration) - .defaultServiceExceptionSupplier(S3Exception::builder) - .build(); - } - - private static URL createTestUrl() { - try { - return new URL(TEST_URL); - } catch (Exception e) { - throw new RuntimeException(e); - } - } - - private void assertSuccessfulGetObject(PresignedUrlGetObjectRequest request) { - try { - ResponseInputStream result = presignedUrlManager.getObject(request, ResponseTransformer.toInputStream()); - assertThat(result).isNotNull(); - String content = IoUtils.toUtf8String(result); - assertThat(content).isEqualTo(TEST_CONTENT); - - SdkHttpRequest lastRequest = mockHttpClient.getLastRequest(); - assertThat(lastRequest.method()).isEqualTo(SdkHttpMethod.GET); - assertThat(lastRequest.getUri().toString()).contains("test-bucket.s3.us-east-1.amazonaws.com/test-key"); - } catch (IOException e) { - throw new RuntimeException(e); - } - } - private static HttpExecuteResponse createSuccessResponse() { - SdkHttpFullResponse httpResponse = SdkHttpFullResponse.builder() - .statusCode(200) - .putHeader("Content-Length", "12") - .putHeader("ETag", "\"test-etag\"") - .putHeader("Content-Type", "text/plain") - .build(); - return HttpExecuteResponse.builder() - .response(httpResponse) - .responseBody(AbortableInputStream.create( - new ByteArrayInputStream(TEST_CONTENT.getBytes(StandardCharsets.UTF_8)))) - .build(); - } - - private static HttpExecuteResponse createErrorResponse(int statusCode, String errorCode, String errorMessage) { - String errorContent = String.format( - "%s%s", - errorCode, errorMessage); - SdkHttpFullResponse httpResponse = SdkHttpFullResponse.builder() - .statusCode(statusCode) - .putHeader("x-amz-request-id", "test-request-id") - .putHeader("x-amz-id-2", "test-extended-request-id") - .build(); - return HttpExecuteResponse.builder() - .response(httpResponse) - .responseBody(AbortableInputStream.create( - new ByteArrayInputStream(errorContent.getBytes(StandardCharsets.UTF_8)))) - .build(); - } -} - From 1994b7f00c74e04194487ab1b1fc48689fe7b1dc Mon Sep 17 00:00:00 2001 From: jencymaryjoseph <35571282+jencymaryjoseph@users.noreply.github.com> Date: Thu, 17 Jul 2025 13:23:25 -0700 Subject: [PATCH 06/30] Add PresignedUrlManager support to code generated S3AsyncClient (#6269) * async s3 client codegen changes * minimised service-2.json --- .../amazon/awssdk/codegen/AddMetadata.java | 1 + .../awssdk/codegen/internal/Constant.java | 2 + .../customization/CustomizationConfig.java | 13 + .../codegen/model/intermediate/Metadata.java | 18 ++ .../codegen/naming/DefaultNamingStrategy.java | 5 + .../awssdk/codegen/naming/NamingStrategy.java | 5 + .../awssdk/codegen/poet/PoetExtension.java | 4 + .../codegen/poet/client/AsyncClientClass.java | 20 ++ .../poet/client/AsyncClientInterface.java | 18 ++ .../client/DelegatingAsyncClientClass.java | 5 + .../awssdk/codegen/poet/ClientTestModels.java | 12 + .../poet/client/AsyncClientClassTest.java | 7 + .../poet/client/AsyncClientInterfaceTest.java | 7 + .../DelegatingAsyncClientClassTest.java | 8 + .../c2j/presignedurl/customization.config | 3 + .../client/c2j/presignedurl/service-2.json | 62 +++++ ...tract-async-client-class-presignedurl.java | 84 +++++++ ...n-async-client-interface-presignedurl.java | 123 ++++++++++ .../poet/client/test-presignedurl-async.java | 230 ++++++++++++++++++ .../codegen-resources/customization.config | 3 +- 20 files changed, 629 insertions(+), 1 deletion(-) create mode 100644 codegen/src/test/resources/software/amazon/awssdk/codegen/poet/client/c2j/presignedurl/customization.config create mode 100644 codegen/src/test/resources/software/amazon/awssdk/codegen/poet/client/c2j/presignedurl/service-2.json create mode 100644 codegen/src/test/resources/software/amazon/awssdk/codegen/poet/client/test-abstract-async-client-class-presignedurl.java create mode 100644 codegen/src/test/resources/software/amazon/awssdk/codegen/poet/client/test-json-async-client-interface-presignedurl.java create mode 100644 codegen/src/test/resources/software/amazon/awssdk/codegen/poet/client/test-presignedurl-async.java diff --git a/codegen/src/main/java/software/amazon/awssdk/codegen/AddMetadata.java b/codegen/src/main/java/software/amazon/awssdk/codegen/AddMetadata.java index 41d1f32693f8..f5764111371c 100644 --- a/codegen/src/main/java/software/amazon/awssdk/codegen/AddMetadata.java +++ b/codegen/src/main/java/software/amazon/awssdk/codegen/AddMetadata.java @@ -62,6 +62,7 @@ public static Metadata constructMetadata(ServiceModel serviceModel, .withDocumentation(serviceModel.getDocumentation()) .withServiceAbbreviation(serviceMetadata.getServiceAbbreviation()) .withBatchmanagerPackageName(namingStrategy.getBatchManagerPackageName(serviceName)) + .withPresignedUrlPackageName(namingStrategy.getPresignedUrlPackageName(serviceName)) .withServiceFullName(serviceMetadata.getServiceFullName()) .withServiceName(serviceName) .withSyncClient(String.format(Constant.SYNC_CLIENT_CLASS_NAME_PATTERN, serviceName)) diff --git a/codegen/src/main/java/software/amazon/awssdk/codegen/internal/Constant.java b/codegen/src/main/java/software/amazon/awssdk/codegen/internal/Constant.java index 5384975f49fb..a0bf024211b7 100644 --- a/codegen/src/main/java/software/amazon/awssdk/codegen/internal/Constant.java +++ b/codegen/src/main/java/software/amazon/awssdk/codegen/internal/Constant.java @@ -78,6 +78,8 @@ public final class Constant { public static final String PACKAGE_NAME_BATCHMANAGER_PATTERN = "%s.batchmanager"; + public static final String PACKAGE_NAME_PRESIGNEDURL_PATTERN = "%s.presignedurl"; + public static final String PACKAGE_NAME_CUSTOM_AUTH_PATTERN = "%s.auth"; public static final String AUTH_POLICY_ENUM_CLASS_DIR = "software/amazon/awssdk/auth/policy/actions"; diff --git a/codegen/src/main/java/software/amazon/awssdk/codegen/model/config/customization/CustomizationConfig.java b/codegen/src/main/java/software/amazon/awssdk/codegen/model/config/customization/CustomizationConfig.java index d3cc24a8eeae..4a7477b40290 100644 --- a/codegen/src/main/java/software/amazon/awssdk/codegen/model/config/customization/CustomizationConfig.java +++ b/codegen/src/main/java/software/amazon/awssdk/codegen/model/config/customization/CustomizationConfig.java @@ -351,6 +351,11 @@ public class CustomizationConfig { */ private boolean batchManagerSupported; + /** + * A boolean flag to indicate if Presigned URL Manager is supported. + */ + private boolean presignedUrlManagerSupported; + /** * A boolean flag to indicate if the fast unmarshaller code path is enabled. */ @@ -946,6 +951,14 @@ public void setBatchManagerSupported(boolean batchManagerSupported) { this.batchManagerSupported = batchManagerSupported; } + public boolean getPresignedUrlManagerSupported() { + return presignedUrlManagerSupported; + } + + public void setPresignedUrlManagerSupported(boolean presignedUrlManagerSupported) { + this.presignedUrlManagerSupported = presignedUrlManagerSupported; + } + public boolean getEnableFastUnmarshaller() { return enableFastUnmarshaller; } diff --git a/codegen/src/main/java/software/amazon/awssdk/codegen/model/intermediate/Metadata.java b/codegen/src/main/java/software/amazon/awssdk/codegen/model/intermediate/Metadata.java index 4eb10f6bf105..3cd8f5df4113 100644 --- a/codegen/src/main/java/software/amazon/awssdk/codegen/model/intermediate/Metadata.java +++ b/codegen/src/main/java/software/amazon/awssdk/codegen/model/intermediate/Metadata.java @@ -73,6 +73,8 @@ public class Metadata { private String batchManagerPackageName; + private String presignedUrlPackageName; + private String endpointRulesPackageName; private String authSchemePackageName; @@ -815,4 +817,20 @@ public String getFullBatchManagerPackageName() { return joinPackageNames(rootPackageName, getBatchManagerPackageName()); } + public Metadata withPresignedUrlPackageName(String presignedUrlPackageName) { + setPresignedUrlPackageName(presignedUrlPackageName); + return this; + } + + public String getPresignedUrlPackageName() { + return presignedUrlPackageName; + } + + public void setPresignedUrlPackageName(String presignedUrlPackageName) { + this.presignedUrlPackageName = presignedUrlPackageName; + } + + public String getFullPresignedUrlPackageName() { + return joinPackageNames(rootPackageName, getPresignedUrlPackageName()); + } } diff --git a/codegen/src/main/java/software/amazon/awssdk/codegen/naming/DefaultNamingStrategy.java b/codegen/src/main/java/software/amazon/awssdk/codegen/naming/DefaultNamingStrategy.java index b9fcf41e6e0f..dc5257d9c9aa 100644 --- a/codegen/src/main/java/software/amazon/awssdk/codegen/naming/DefaultNamingStrategy.java +++ b/codegen/src/main/java/software/amazon/awssdk/codegen/naming/DefaultNamingStrategy.java @@ -220,6 +220,11 @@ public String getJmesPathPackageName(String serviceName) { public String getBatchManagerPackageName(String serviceName) { return getCustomizedPackageName(concatServiceNameIfShareModel(serviceName), Constant.PACKAGE_NAME_BATCHMANAGER_PATTERN); } + + @Override + public String getPresignedUrlPackageName(String serviceName) { + return getCustomizedPackageName(concatServiceNameIfShareModel(serviceName), Constant.PACKAGE_NAME_PRESIGNEDURL_PATTERN); + } @Override public String getSmokeTestPackageName(String serviceName) { diff --git a/codegen/src/main/java/software/amazon/awssdk/codegen/naming/NamingStrategy.java b/codegen/src/main/java/software/amazon/awssdk/codegen/naming/NamingStrategy.java index 5b22363970f0..1047a3a1904b 100644 --- a/codegen/src/main/java/software/amazon/awssdk/codegen/naming/NamingStrategy.java +++ b/codegen/src/main/java/software/amazon/awssdk/codegen/naming/NamingStrategy.java @@ -93,6 +93,11 @@ public interface NamingStrategy { * Retrieve the batchManager package name that should be used based on the service name. */ String getBatchManagerPackageName(String serviceName); + + /** + * Retrieve the presignedUrl package name that should be used based on the service name. + */ + String getPresignedUrlPackageName(String serviceName); /** * Retrieve the smote test package name that should be used based on the service name. diff --git a/codegen/src/main/java/software/amazon/awssdk/codegen/poet/PoetExtension.java b/codegen/src/main/java/software/amazon/awssdk/codegen/poet/PoetExtension.java index f119507809c2..54888ce41b57 100644 --- a/codegen/src/main/java/software/amazon/awssdk/codegen/poet/PoetExtension.java +++ b/codegen/src/main/java/software/amazon/awssdk/codegen/poet/PoetExtension.java @@ -209,4 +209,8 @@ public ClassName getBatchManagerAsyncInterface() { return ClassName.get(model.getMetadata().getFullBatchManagerPackageName(), model.getMetadata().getServiceName() + "AsyncBatchManager"); } + + public ClassName getPresignedUrlManagerAsyncInterface() { + return ClassName.get(model.getMetadata().getFullPresignedUrlPackageName(), "AsyncPresignedUrlManager"); + } } diff --git a/codegen/src/main/java/software/amazon/awssdk/codegen/poet/client/AsyncClientClass.java b/codegen/src/main/java/software/amazon/awssdk/codegen/poet/client/AsyncClientClass.java index 539d5df35a43..d29c3124475b 100644 --- a/codegen/src/main/java/software/amazon/awssdk/codegen/poet/client/AsyncClientClass.java +++ b/codegen/src/main/java/software/amazon/awssdk/codegen/poet/client/AsyncClientClass.java @@ -538,6 +538,26 @@ protected void addBatchManagerMethod(Builder type) { type.addMethod(batchManager); } + + @Override + protected void addPresignedUrlManagerMethod(Builder type) { + ClassName returnType = poetExtensions.getPresignedUrlManagerAsyncInterface(); + String internalPresignedUrlPackage = model.getMetadata().getFullInternalPackageName() + ".presignedurl"; + ClassName implClass = ClassName.get(internalPresignedUrlPackage, "DefaultAsyncPresignedUrlManager"); + + MethodSpec presignedUrlManager = MethodSpec.methodBuilder("presignedUrlManager") + .addModifiers(PUBLIC) + .addAnnotation(Override.class) + .returns(returnType) + .addStatement("return new $T(clientHandler," + + " protocolFactory, " + + "clientConfiguration," + + " protocolMetadata)", + implClass) + .build(); + + type.addMethod(presignedUrlManager); + } private MethodSpec resolveMetricPublishersMethod() { String clientConfigName = "clientConfiguration"; diff --git a/codegen/src/main/java/software/amazon/awssdk/codegen/poet/client/AsyncClientInterface.java b/codegen/src/main/java/software/amazon/awssdk/codegen/poet/client/AsyncClientInterface.java index cbd5f3cb3e9a..ab4e9223ff40 100644 --- a/codegen/src/main/java/software/amazon/awssdk/codegen/poet/client/AsyncClientInterface.java +++ b/codegen/src/main/java/software/amazon/awssdk/codegen/poet/client/AsyncClientInterface.java @@ -99,6 +99,9 @@ public TypeSpec poetSpec() { if (model.getCustomizationConfig().getBatchManagerSupported()) { addBatchManagerMethod(result); } + if (model.getCustomizationConfig().getPresignedUrlManagerSupported()) { + addPresignedUrlManagerMethod(result); + } result.addMethod(serviceClientConfigMethod()); addAdditionalMethods(result); addCloseMethod(result); @@ -174,6 +177,16 @@ protected void addBatchManagerMethod(TypeSpec.Builder type) { + "configuration set on this client.", returnType); type.addMethod(batchManagerOperationBody(builder).build()); } + + protected void addPresignedUrlManagerMethod(TypeSpec.Builder type) { + ClassName returnType = poetExtensions.getPresignedUrlManagerAsyncInterface(); + MethodSpec.Builder builder = MethodSpec.methodBuilder("presignedUrlManager") + .addModifiers(PUBLIC) + .returns(returnType) + .addJavadoc("Creates an instance of {@link $T} object with the " + + "configuration set on this client.", returnType); + type.addMethod(presignedUrlManagerOperationBody(builder).build()); + } @Override public ClassName className() { @@ -550,5 +563,10 @@ protected MethodSpec.Builder batchManagerOperationBody(MethodSpec.Builder builde return builder.addModifiers(DEFAULT, PUBLIC) .addStatement("throw new $T()", UnsupportedOperationException.class); } + + protected MethodSpec.Builder presignedUrlManagerOperationBody(MethodSpec.Builder builder) { + return builder.addModifiers(DEFAULT, PUBLIC) + .addStatement("throw new $T()", UnsupportedOperationException.class); + } } diff --git a/codegen/src/main/java/software/amazon/awssdk/codegen/poet/client/DelegatingAsyncClientClass.java b/codegen/src/main/java/software/amazon/awssdk/codegen/poet/client/DelegatingAsyncClientClass.java index 5af2c45ad0bc..be0239e00808 100644 --- a/codegen/src/main/java/software/amazon/awssdk/codegen/poet/client/DelegatingAsyncClientClass.java +++ b/codegen/src/main/java/software/amazon/awssdk/codegen/poet/client/DelegatingAsyncClientClass.java @@ -222,4 +222,9 @@ protected MethodSpec.Builder waiterOperationBody(MethodSpec.Builder builder) { protected MethodSpec.Builder batchManagerOperationBody(MethodSpec.Builder builder) { return builder.addAnnotation(Override.class).addStatement("return delegate.batchManager()"); } + + @Override + protected MethodSpec.Builder presignedUrlManagerOperationBody(MethodSpec.Builder builder) { + return builder.addAnnotation(Override.class).addStatement("return delegate.presignedUrlManager()"); + } } diff --git a/codegen/src/test/java/software/amazon/awssdk/codegen/poet/ClientTestModels.java b/codegen/src/test/java/software/amazon/awssdk/codegen/poet/ClientTestModels.java index acd08400564a..82a80e58652b 100644 --- a/codegen/src/test/java/software/amazon/awssdk/codegen/poet/ClientTestModels.java +++ b/codegen/src/test/java/software/amazon/awssdk/codegen/poet/ClientTestModels.java @@ -605,6 +605,18 @@ public static IntermediateModel batchManagerModels() { return new IntermediateModelBuilder(models).build(); } + + public static IntermediateModel presignedUrlManagerModels() { + File serviceModel = new File(ClientTestModels.class.getResource("client/c2j/presignedurl/service-2.json").getFile()); + File customizationModel = new File(ClientTestModels.class.getResource("client/c2j/presignedurl/customization.config").getFile()); + + C2jModels models = C2jModels.builder() + .serviceModel(getServiceModel(serviceModel)) + .customizationConfig(getCustomizationConfig(customizationModel)) + .build(); + + return new IntermediateModelBuilder(models).build(); + } private static ServiceModel getServiceModel(File file) { return ModelLoaderUtils.loadModel(ServiceModel.class, file); diff --git a/codegen/src/test/java/software/amazon/awssdk/codegen/poet/client/AsyncClientClassTest.java b/codegen/src/test/java/software/amazon/awssdk/codegen/poet/client/AsyncClientClassTest.java index 6e2082fc4175..7a7c74375cb9 100644 --- a/codegen/src/test/java/software/amazon/awssdk/codegen/poet/client/AsyncClientClassTest.java +++ b/codegen/src/test/java/software/amazon/awssdk/codegen/poet/client/AsyncClientClassTest.java @@ -24,6 +24,7 @@ import static software.amazon.awssdk.codegen.poet.ClientTestModels.customPackageModels; import static software.amazon.awssdk.codegen.poet.ClientTestModels.endpointDiscoveryModels; import static software.amazon.awssdk.codegen.poet.ClientTestModels.opsWithSigv4a; +import static software.amazon.awssdk.codegen.poet.ClientTestModels.presignedUrlManagerModels; import static software.amazon.awssdk.codegen.poet.ClientTestModels.serviceWithCustomContextParamsModels; import static software.amazon.awssdk.codegen.poet.ClientTestModels.queryServiceModels; import static software.amazon.awssdk.codegen.poet.ClientTestModels.restJsonServiceModels; @@ -102,6 +103,12 @@ public void asyncClientBatchManager() { ClassSpec aSyncClientBatchManager = createAsyncClientClass(batchManagerModels()); assertThat(aSyncClientBatchManager, generatesTo("test-batchmanager-async.java")); } + + @Test + public void asyncClientPresignedUrlManager() { + ClassSpec asyncClientPresignedUrlManager = createAsyncClientClass(presignedUrlManagerModels()); + assertThat(asyncClientPresignedUrlManager, generatesTo("test-presignedurl-async.java")); + } @Test public void asyncClientWithStreamingUnsignedPayload() { diff --git a/codegen/src/test/java/software/amazon/awssdk/codegen/poet/client/AsyncClientInterfaceTest.java b/codegen/src/test/java/software/amazon/awssdk/codegen/poet/client/AsyncClientInterfaceTest.java index fda4c6f2b459..1a2f18183e8d 100644 --- a/codegen/src/test/java/software/amazon/awssdk/codegen/poet/client/AsyncClientInterfaceTest.java +++ b/codegen/src/test/java/software/amazon/awssdk/codegen/poet/client/AsyncClientInterfaceTest.java @@ -17,6 +17,7 @@ import static org.hamcrest.MatcherAssert.assertThat; import static software.amazon.awssdk.codegen.poet.ClientTestModels.batchManagerModels; +import static software.amazon.awssdk.codegen.poet.ClientTestModels.presignedUrlManagerModels; import static software.amazon.awssdk.codegen.poet.ClientTestModels.restJsonServiceModels; import static software.amazon.awssdk.codegen.poet.PoetMatchers.generatesTo; @@ -35,4 +36,10 @@ public void asyncClientInterfaceWithBatchManager() { ClassSpec asyncClientInterface = new AsyncClientInterface(batchManagerModels()); assertThat(asyncClientInterface, generatesTo("test-json-async-client-interface-batchmanager.java")); } + + @Test + public void asyncClientInterfaceWithPresignedUrlManager() { + ClassSpec asyncClientInterface = new AsyncClientInterface(presignedUrlManagerModels()); + assertThat(asyncClientInterface, generatesTo("test-json-async-client-interface-presignedurl.java")); + } } diff --git a/codegen/src/test/java/software/amazon/awssdk/codegen/poet/client/DelegatingAsyncClientClassTest.java b/codegen/src/test/java/software/amazon/awssdk/codegen/poet/client/DelegatingAsyncClientClassTest.java index a12ab3ac659c..a77cb67ab727 100644 --- a/codegen/src/test/java/software/amazon/awssdk/codegen/poet/client/DelegatingAsyncClientClassTest.java +++ b/codegen/src/test/java/software/amazon/awssdk/codegen/poet/client/DelegatingAsyncClientClassTest.java @@ -16,6 +16,7 @@ package software.amazon.awssdk.codegen.poet.client; import static org.hamcrest.MatcherAssert.assertThat; +import static software.amazon.awssdk.codegen.poet.ClientTestModels.presignedUrlManagerModels; import static software.amazon.awssdk.codegen.poet.ClientTestModels.restJsonServiceModels; import static software.amazon.awssdk.codegen.poet.PoetMatchers.generatesTo; @@ -28,4 +29,11 @@ public void delegatingAsyncClientClass() { new DelegatingAsyncClientClass(restJsonServiceModels()); assertThat(asyncClientDecoratorAbstractClass, generatesTo("test-abstract-async-client-class.java")); } + + @Test + public void delegatingAsyncClientClassWithPresignedUrlManager() { + DelegatingAsyncClientClass asyncClientDecoratorAbstractClass = + new DelegatingAsyncClientClass(presignedUrlManagerModels()); + assertThat(asyncClientDecoratorAbstractClass, generatesTo("test-abstract-async-client-class-presignedurl.java")); + } } diff --git a/codegen/src/test/resources/software/amazon/awssdk/codegen/poet/client/c2j/presignedurl/customization.config b/codegen/src/test/resources/software/amazon/awssdk/codegen/poet/client/c2j/presignedurl/customization.config new file mode 100644 index 000000000000..cfc5b2874b7a --- /dev/null +++ b/codegen/src/test/resources/software/amazon/awssdk/codegen/poet/client/c2j/presignedurl/customization.config @@ -0,0 +1,3 @@ +{ + "presignedUrlManagerSupported": true +} \ No newline at end of file diff --git a/codegen/src/test/resources/software/amazon/awssdk/codegen/poet/client/c2j/presignedurl/service-2.json b/codegen/src/test/resources/software/amazon/awssdk/codegen/poet/client/c2j/presignedurl/service-2.json new file mode 100644 index 000000000000..cfeb496ac347 --- /dev/null +++ b/codegen/src/test/resources/software/amazon/awssdk/codegen/poet/client/c2j/presignedurl/service-2.json @@ -0,0 +1,62 @@ +{ + "version": "2.0", + "metadata": { + "apiVersion": "2010-05-08", + "endpointPrefix": "json-service-endpoint", + "globalEndpoint": "json-service.amazonaws.com", + "protocol": "json", + "serviceAbbreviation": "Json Service", + "serviceFullName": "Some Service That Uses AWS Json Protocol", + "serviceId": "Json Service", + "signingName": "json-service", + "signatureVersion": "v4", + "uid": "json-service-2010-05-08", + "xmlNamespace": "https://json-service.amazonaws.com/doc/2010-05-08/" + }, + "operations": { + "APostOperation": { + "name": "APostOperation", + "http": { + "method": "POST", + "requestUri": "/" + }, + "input": { + "shape": "APostOperationRequest" + }, + "errors": [ + { + "shape": "InvalidInputException" + } + ], + "documentation": "

Performs a post operation to the query service and has no output

" + } + }, + "shapes": { + "APostOperationRequest": { + "type": "structure", + "required": [ + "StringMember" + ], + "members": { + "StringMember": { + "shape": "String", + "documentation": "

String member

" + } + } + }, + "InvalidInputException": { + "type": "structure", + "members": { + "message": { + "shape": "String" + } + }, + "documentation": "

The request was rejected because an invalid or out-of-range value was supplied for an input parameter.

", + "exception": true + }, + "String": { + "type": "string" + } + }, + "documentation": "A service that is implemented using the json protocol" +} \ No newline at end of file diff --git a/codegen/src/test/resources/software/amazon/awssdk/codegen/poet/client/test-abstract-async-client-class-presignedurl.java b/codegen/src/test/resources/software/amazon/awssdk/codegen/poet/client/test-abstract-async-client-class-presignedurl.java new file mode 100644 index 000000000000..715c56f752a7 --- /dev/null +++ b/codegen/src/test/resources/software/amazon/awssdk/codegen/poet/client/test-abstract-async-client-class-presignedurl.java @@ -0,0 +1,84 @@ +package software.amazon.awssdk.services.json; + +import java.util.concurrent.CompletableFuture; +import java.util.function.Function; +import software.amazon.awssdk.annotations.Generated; +import software.amazon.awssdk.annotations.SdkPublicApi; +import software.amazon.awssdk.core.SdkClient; +import software.amazon.awssdk.services.json.model.APostOperationRequest; +import software.amazon.awssdk.services.json.model.APostOperationResponse; +import software.amazon.awssdk.services.json.model.JsonRequest; +import software.amazon.awssdk.services.json.presignedurl.AsyncPresignedUrlManager; +import software.amazon.awssdk.utils.Validate; + +@Generated("software.amazon.awssdk:codegen") +@SdkPublicApi +public abstract class DelegatingJsonAsyncClient implements JsonAsyncClient { + private final JsonAsyncClient delegate; + + public DelegatingJsonAsyncClient(JsonAsyncClient delegate) { + Validate.paramNotNull(delegate, "delegate"); + this.delegate = delegate; + } + + /** + *

+ * Performs a post operation to the query service and has no output + *

+ * + * @param aPostOperationRequest + * @return A Java Future containing the result of the APostOperation operation returned by the service.
+ * The CompletableFuture returned by this method can be completed exceptionally with the following + * exceptions. The exception returned is wrapped with CompletionException, so you need to invoke + * {@link Throwable#getCause} to retrieve the underlying exception. + *
    + *
  • InvalidInputException The request was rejected because an invalid or out-of-range value was supplied + * for an input parameter.
    • + *
  • SdkException Base class for all exceptions that can be thrown by the SDK (both service and client). + * Can be used for catch all scenarios.
    • + *
  • SdkClientException If any client side error occurs such as an IO related failure, failure to get + * credentials, etc.
    • + *
  • JsonException Base class for all service exceptions. Unknown exceptions will be thrown as an instance + * of this type.
    • + *
+ * @sample JsonAsyncClient.APostOperation + * @see AWS + * API Documentation + */ + @Override + public CompletableFuture aPostOperation(APostOperationRequest aPostOperationRequest) { + return invokeOperation(aPostOperationRequest, request -> delegate.aPostOperation(request)); + } + + /** + * Creates an instance of {@link AsyncPresignedUrlManager} object with the configuration set on this client. + */ + @Override + public AsyncPresignedUrlManager presignedUrlManager() { + return delegate.presignedUrlManager(); + } + + @Override + public final JsonServiceClientConfiguration serviceClientConfiguration() { + return delegate.serviceClientConfiguration(); + } + + @Override + public final String serviceName() { + return delegate.serviceName(); + } + + public SdkClient delegate() { + return this.delegate; + } + + protected CompletableFuture invokeOperation(T request, + Function> operation) { + return operation.apply(request); + } + + @Override + public void close() { + delegate.close(); + } +} diff --git a/codegen/src/test/resources/software/amazon/awssdk/codegen/poet/client/test-json-async-client-interface-presignedurl.java b/codegen/src/test/resources/software/amazon/awssdk/codegen/poet/client/test-json-async-client-interface-presignedurl.java new file mode 100644 index 000000000000..631198b323bd --- /dev/null +++ b/codegen/src/test/resources/software/amazon/awssdk/codegen/poet/client/test-json-async-client-interface-presignedurl.java @@ -0,0 +1,123 @@ +package software.amazon.awssdk.services.json; + +import java.util.concurrent.CompletableFuture; +import java.util.function.Consumer; +import software.amazon.awssdk.annotations.Generated; +import software.amazon.awssdk.annotations.SdkPublicApi; +import software.amazon.awssdk.annotations.ThreadSafe; +import software.amazon.awssdk.awscore.AwsClient; +import software.amazon.awssdk.services.json.model.APostOperationRequest; +import software.amazon.awssdk.services.json.model.APostOperationResponse; +import software.amazon.awssdk.services.json.presignedurl.AsyncPresignedUrlManager; + +/** + * Service client for accessing Json Service asynchronously. This can be created using the static {@link #builder()} + * method.The asynchronous client performs non-blocking I/O when configured with any {@code SdkAsyncHttpClient} + * supported in the SDK. However, full non-blocking is not guaranteed as the async client may perform blocking calls in + * some cases such as credentials retrieval and endpoint discovery as part of the async API call. + * + * A service that is implemented using the json protocol + */ +@Generated("software.amazon.awssdk:codegen") +@SdkPublicApi +@ThreadSafe +public interface JsonAsyncClient extends AwsClient { + String SERVICE_NAME = "json-service"; + + /** + * Value for looking up the service's metadata from the + * {@link software.amazon.awssdk.regions.ServiceMetadataProvider}. + */ + String SERVICE_METADATA_ID = "json-service-endpoint"; + + /** + *

+ * Performs a post operation to the query service and has no output + *

+ * + * @param aPostOperationRequest + * @return A Java Future containing the result of the APostOperation operation returned by the service.
+ * The CompletableFuture returned by this method can be completed exceptionally with the following + * exceptions. The exception returned is wrapped with CompletionException, so you need to invoke + * {@link Throwable#getCause} to retrieve the underlying exception. + *
    + *
  • InvalidInputException The request was rejected because an invalid or out-of-range value was supplied + * for an input parameter.
    • + *
  • SdkException Base class for all exceptions that can be thrown by the SDK (both service and client). + * Can be used for catch all scenarios.
    • + *
  • SdkClientException If any client side error occurs such as an IO related failure, failure to get + * credentials, etc.
    • + *
  • JsonException Base class for all service exceptions. Unknown exceptions will be thrown as an instance + * of this type.
    • + *
+ * @sample JsonAsyncClient.APostOperation + * @see AWS + * API Documentation + */ + default CompletableFuture aPostOperation(APostOperationRequest aPostOperationRequest) { + throw new UnsupportedOperationException(); + } + + /** + *

+ * Performs a post operation to the query service and has no output + *

+ *
+ *

+ * This is a convenience which creates an instance of the {@link APostOperationRequest.Builder} avoiding the need to + * create one manually via {@link APostOperationRequest#builder()} + *

+ * + * @param aPostOperationRequest + * A {@link Consumer} that will call methods on + * {@link software.amazon.awssdk.services.json.model.APostOperationRequest.Builder} to create a request. + * @return A Java Future containing the result of the APostOperation operation returned by the service.
+ * The CompletableFuture returned by this method can be completed exceptionally with the following + * exceptions. The exception returned is wrapped with CompletionException, so you need to invoke + * {@link Throwable#getCause} to retrieve the underlying exception. + *
    + *
  • InvalidInputException The request was rejected because an invalid or out-of-range value was supplied + * for an input parameter.
    • + *
  • SdkException Base class for all exceptions that can be thrown by the SDK (both service and client). + * Can be used for catch all scenarios.
    • + *
  • SdkClientException If any client side error occurs such as an IO related failure, failure to get + * credentials, etc.
    • + *
  • JsonException Base class for all service exceptions. Unknown exceptions will be thrown as an instance + * of this type.
    • + *
+ * @sample JsonAsyncClient.APostOperation + * @see AWS + * API Documentation + */ + default CompletableFuture aPostOperation(Consumer aPostOperationRequest) { + return aPostOperation(APostOperationRequest.builder().applyMutation(aPostOperationRequest).build()); + } + + /** + * Creates an instance of {@link AsyncPresignedUrlManager} object with the configuration set on this client. + */ + default AsyncPresignedUrlManager presignedUrlManager() { + throw new UnsupportedOperationException(); + } + + @Override + default JsonServiceClientConfiguration serviceClientConfiguration() { + throw new UnsupportedOperationException(); + } + + /** + * Create a {@link JsonAsyncClient} with the region loaded from the + * {@link software.amazon.awssdk.regions.providers.DefaultAwsRegionProviderChain} and credentials loaded from the + * {@link software.amazon.awssdk.auth.credentials.DefaultCredentialsProvider}. + */ + static JsonAsyncClient create() { + return builder().build(); + } + + /** + * Create a builder that can be used to configure and create a {@link JsonAsyncClient}. + */ + static JsonAsyncClientBuilder builder() { + return new DefaultJsonAsyncClientBuilder(); + } +} diff --git a/codegen/src/test/resources/software/amazon/awssdk/codegen/poet/client/test-presignedurl-async.java b/codegen/src/test/resources/software/amazon/awssdk/codegen/poet/client/test-presignedurl-async.java new file mode 100644 index 000000000000..5afef2048e82 --- /dev/null +++ b/codegen/src/test/resources/software/amazon/awssdk/codegen/poet/client/test-presignedurl-async.java @@ -0,0 +1,230 @@ +package software.amazon.awssdk.services.json; + +import static software.amazon.awssdk.utils.FunctionalUtils.runAndLogError; + +import java.util.Collections; +import java.util.List; +import java.util.Optional; +import java.util.concurrent.CompletableFuture; +import java.util.function.Consumer; +import java.util.function.Function; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import software.amazon.awssdk.annotations.Generated; +import software.amazon.awssdk.annotations.SdkInternalApi; +import software.amazon.awssdk.awscore.client.handler.AwsAsyncClientHandler; +import software.amazon.awssdk.awscore.exception.AwsServiceException; +import software.amazon.awssdk.awscore.internal.AwsProtocolMetadata; +import software.amazon.awssdk.awscore.internal.AwsServiceProtocol; +import software.amazon.awssdk.awscore.retry.AwsRetryStrategy; +import software.amazon.awssdk.core.RequestOverrideConfiguration; +import software.amazon.awssdk.core.SdkPlugin; +import software.amazon.awssdk.core.SdkRequest; +import software.amazon.awssdk.core.client.config.ClientOverrideConfiguration; +import software.amazon.awssdk.core.client.config.SdkClientConfiguration; +import software.amazon.awssdk.core.client.config.SdkClientOption; +import software.amazon.awssdk.core.client.handler.AsyncClientHandler; +import software.amazon.awssdk.core.client.handler.ClientExecutionParams; +import software.amazon.awssdk.core.http.HttpResponseHandler; +import software.amazon.awssdk.core.metrics.CoreMetric; +import software.amazon.awssdk.core.retry.RetryMode; +import software.amazon.awssdk.metrics.MetricCollector; +import software.amazon.awssdk.metrics.MetricPublisher; +import software.amazon.awssdk.metrics.NoOpMetricCollector; +import software.amazon.awssdk.protocols.core.ExceptionMetadata; +import software.amazon.awssdk.protocols.json.AwsJsonProtocol; +import software.amazon.awssdk.protocols.json.AwsJsonProtocolFactory; +import software.amazon.awssdk.protocols.json.BaseAwsJsonProtocolFactory; +import software.amazon.awssdk.protocols.json.JsonOperationMetadata; +import software.amazon.awssdk.retries.api.RetryStrategy; +import software.amazon.awssdk.services.json.internal.JsonServiceClientConfigurationBuilder; +import software.amazon.awssdk.services.json.internal.ServiceVersionInfo; +import software.amazon.awssdk.services.json.internal.presignedurl.DefaultAsyncPresignedUrlManager; +import software.amazon.awssdk.services.json.model.APostOperationRequest; +import software.amazon.awssdk.services.json.model.APostOperationResponse; +import software.amazon.awssdk.services.json.model.InvalidInputException; +import software.amazon.awssdk.services.json.model.JsonException; +import software.amazon.awssdk.services.json.presignedurl.AsyncPresignedUrlManager; +import software.amazon.awssdk.services.json.transform.APostOperationRequestMarshaller; +import software.amazon.awssdk.utils.CompletableFutureUtils; + +/** + * Internal implementation of {@link JsonAsyncClient}. + * + * @see JsonAsyncClient#builder() + */ +@Generated("software.amazon.awssdk:codegen") +@SdkInternalApi +final class DefaultJsonAsyncClient implements JsonAsyncClient { + private static final Logger log = LoggerFactory.getLogger(DefaultJsonAsyncClient.class); + + private static final AwsProtocolMetadata protocolMetadata = AwsProtocolMetadata.builder() + .serviceProtocol(AwsServiceProtocol.AWS_JSON).build(); + + private final AsyncClientHandler clientHandler; + + private final AwsJsonProtocolFactory protocolFactory; + + private final SdkClientConfiguration clientConfiguration; + + protected DefaultJsonAsyncClient(SdkClientConfiguration clientConfiguration) { + this.clientHandler = new AwsAsyncClientHandler(clientConfiguration); + this.clientConfiguration = clientConfiguration.toBuilder().option(SdkClientOption.SDK_CLIENT, this) + .option(SdkClientOption.API_METADATA, "Json_Service" + "#" + ServiceVersionInfo.VERSION).build(); + this.protocolFactory = init(AwsJsonProtocolFactory.builder()).build(); + } + + /** + *

+ * Performs a post operation to the query service and has no output + *

+ * + * @param aPostOperationRequest + * @return A Java Future containing the result of the APostOperation operation returned by the service.
+ * The CompletableFuture returned by this method can be completed exceptionally with the following + * exceptions. The exception returned is wrapped with CompletionException, so you need to invoke + * {@link Throwable#getCause} to retrieve the underlying exception. + *
    + *
  • InvalidInputException The request was rejected because an invalid or out-of-range value was supplied + * for an input parameter.
    • + *
  • SdkException Base class for all exceptions that can be thrown by the SDK (both service and client). + * Can be used for catch all scenarios.
    • + *
  • SdkClientException If any client side error occurs such as an IO related failure, failure to get + * credentials, etc.
    • + *
  • JsonException Base class for all service exceptions. Unknown exceptions will be thrown as an instance + * of this type.
    • + *
+ * @sample JsonAsyncClient.APostOperation + * @see AWS + * API Documentation + */ + @Override + public CompletableFuture aPostOperation(APostOperationRequest aPostOperationRequest) { + SdkClientConfiguration clientConfiguration = updateSdkClientConfiguration(aPostOperationRequest, this.clientConfiguration); + List metricPublishers = resolveMetricPublishers(clientConfiguration, aPostOperationRequest + .overrideConfiguration().orElse(null)); + MetricCollector apiCallMetricCollector = metricPublishers.isEmpty() ? NoOpMetricCollector.create() : MetricCollector + .create("ApiCall"); + try { + apiCallMetricCollector.reportMetric(CoreMetric.SERVICE_ID, "Json Service"); + apiCallMetricCollector.reportMetric(CoreMetric.OPERATION_NAME, "APostOperation"); + JsonOperationMetadata operationMetadata = JsonOperationMetadata.builder().hasStreamingSuccessResponse(false) + .isPayloadJson(true).build(); + + HttpResponseHandler responseHandler = protocolFactory.createResponseHandler( + operationMetadata, APostOperationResponse::builder); + Function> exceptionMetadataMapper = errorCode -> { + if (errorCode == null) { + return Optional.empty(); + } + switch (errorCode) { + case "InvalidInputException": + return Optional.of(ExceptionMetadata.builder().errorCode("InvalidInputException").httpStatusCode(400) + .exceptionBuilderSupplier(InvalidInputException::builder).build()); + default: + return Optional.empty(); + } + }; + HttpResponseHandler errorResponseHandler = createErrorResponseHandler(protocolFactory, + operationMetadata, exceptionMetadataMapper); + + CompletableFuture executeFuture = clientHandler + .execute(new ClientExecutionParams() + .withOperationName("APostOperation").withProtocolMetadata(protocolMetadata) + .withMarshaller(new APostOperationRequestMarshaller(protocolFactory)) + .withResponseHandler(responseHandler).withErrorResponseHandler(errorResponseHandler) + .withRequestConfiguration(clientConfiguration).withMetricCollector(apiCallMetricCollector) + .withInput(aPostOperationRequest)); + CompletableFuture whenCompleted = executeFuture.whenComplete((r, e) -> { + metricPublishers.forEach(p -> p.publish(apiCallMetricCollector.collect())); + }); + executeFuture = CompletableFutureUtils.forwardExceptionTo(whenCompleted, executeFuture); + return executeFuture; + } catch (Throwable t) { + metricPublishers.forEach(p -> p.publish(apiCallMetricCollector.collect())); + return CompletableFutureUtils.failedFuture(t); + } + } + + @Override + public AsyncPresignedUrlManager presignedUrlManager() { + return new DefaultAsyncPresignedUrlManager(clientHandler, protocolFactory, clientConfiguration, protocolMetadata); + } + + @Override + public final JsonServiceClientConfiguration serviceClientConfiguration() { + return new JsonServiceClientConfigurationBuilder(this.clientConfiguration.toBuilder()).build(); + } + + @Override + public final String serviceName() { + return SERVICE_NAME; + } + + private > T init(T builder) { + return builder.clientConfiguration(clientConfiguration).defaultServiceExceptionSupplier(JsonException::builder) + .protocol(AwsJsonProtocol.AWS_JSON).protocolVersion("1.1"); + } + + private static List resolveMetricPublishers(SdkClientConfiguration clientConfiguration, + RequestOverrideConfiguration requestOverrideConfiguration) { + List publishers = null; + if (requestOverrideConfiguration != null) { + publishers = requestOverrideConfiguration.metricPublishers(); + } + if (publishers == null || publishers.isEmpty()) { + publishers = clientConfiguration.option(SdkClientOption.METRIC_PUBLISHERS); + } + if (publishers == null) { + publishers = Collections.emptyList(); + } + return publishers; + } + + private void updateRetryStrategyClientConfiguration(SdkClientConfiguration.Builder configuration) { + ClientOverrideConfiguration.Builder builder = configuration.asOverrideConfigurationBuilder(); + RetryMode retryMode = builder.retryMode(); + if (retryMode != null) { + configuration.option(SdkClientOption.RETRY_STRATEGY, AwsRetryStrategy.forRetryMode(retryMode)); + } else { + Consumer> configurator = builder.retryStrategyConfigurator(); + if (configurator != null) { + RetryStrategy.Builder defaultBuilder = AwsRetryStrategy.defaultRetryStrategy().toBuilder(); + configurator.accept(defaultBuilder); + configuration.option(SdkClientOption.RETRY_STRATEGY, defaultBuilder.build()); + } else { + RetryStrategy retryStrategy = builder.retryStrategy(); + if (retryStrategy != null) { + configuration.option(SdkClientOption.RETRY_STRATEGY, retryStrategy); + } + } + } + configuration.option(SdkClientOption.CONFIGURED_RETRY_MODE, null); + configuration.option(SdkClientOption.CONFIGURED_RETRY_STRATEGY, null); + configuration.option(SdkClientOption.CONFIGURED_RETRY_CONFIGURATOR, null); + } + + private SdkClientConfiguration updateSdkClientConfiguration(SdkRequest request, SdkClientConfiguration clientConfiguration) { + List plugins = request.overrideConfiguration().map(c -> c.plugins()).orElse(Collections.emptyList()); + SdkClientConfiguration.Builder configuration = clientConfiguration.toBuilder(); + if (plugins.isEmpty()) { + return configuration.build(); + } + JsonServiceClientConfigurationBuilder serviceConfigBuilder = new JsonServiceClientConfigurationBuilder(configuration); + for (SdkPlugin plugin : plugins) { + plugin.configureClient(serviceConfigBuilder); + } + updateRetryStrategyClientConfiguration(configuration); + return configuration.build(); + } + + private HttpResponseHandler createErrorResponseHandler(BaseAwsJsonProtocolFactory protocolFactory, + JsonOperationMetadata operationMetadata, Function> exceptionMetadataMapper) { + return protocolFactory.createErrorResponseHandler(operationMetadata, exceptionMetadataMapper); + } + + @Override + public void close() { + clientHandler.close(); + } +} diff --git a/services/s3/src/main/resources/codegen-resources/customization.config b/services/s3/src/main/resources/codegen-resources/customization.config index 4dee2d9e88d9..0d9b8038bd1f 100644 --- a/services/s3/src/main/resources/codegen-resources/customization.config +++ b/services/s3/src/main/resources/codegen-resources/customization.config @@ -358,5 +358,6 @@ "methodName": "modifyUploadPartCopyRequest", "className": "software.amazon.awssdk.services.s3.internal.CustomRequestTransformerUtils" } - } + }, + "presignedUrlManagerSupported": true } From 2b23126cf099770c6d9a3c1a5803f64a1d575538 Mon Sep 17 00:00:00 2001 From: jencymaryjoseph <35571282+jencymaryjoseph@users.noreply.github.com> Date: Fri, 18 Jul 2025 15:19:25 -0700 Subject: [PATCH 07/30] Modify Operation Name for Presigned URL, add Null Check for ResolvedEndpoint, enhance AsyncPresignedUrlManager API (#6275) * modify operation name, null check for resolvedendpoint, add getObject APIs * added null check for resolvedendpointHeader, doc updates, test additions * fixed checkstyle issue --- .../EndpointResolverInterceptorSpec.java | 3 +- .../endpoint-resolve-interceptor-preSra.java | 2 +- ...e-interceptor-with-endpointsbasedauth.java | 2 +- ...olve-interceptor-with-multiauthsigv4a.java | 2 +- .../rules/endpoint-resolve-interceptor.java | 2 +- .../DefaultAsyncPresignedUrlManager.java | 4 +- .../AsyncPresignedUrlManager.java | 111 ++-- .../DefaultAsyncPresignedUrlManagerTest.java | 2 +- .../AsyncPresignedUrlManagerTest.java | 522 ++++++++++++++++++ 9 files changed, 608 insertions(+), 42 deletions(-) create mode 100644 services/s3/src/test/java/software/amazon/awssdk/services/s3/presignedurl/AsyncPresignedUrlManagerTest.java diff --git a/codegen/src/main/java/software/amazon/awssdk/codegen/poet/rules/EndpointResolverInterceptorSpec.java b/codegen/src/main/java/software/amazon/awssdk/codegen/poet/rules/EndpointResolverInterceptorSpec.java index 726d61cdb99e..57d861ebb025 100644 --- a/codegen/src/main/java/software/amazon/awssdk/codegen/poet/rules/EndpointResolverInterceptorSpec.java +++ b/codegen/src/main/java/software/amazon/awssdk/codegen/poet/rules/EndpointResolverInterceptorSpec.java @@ -255,7 +255,8 @@ private MethodSpec modifyHttpRequestMethod() { b.addStatement("$T resolvedEndpoint = executionAttributes.getAttribute($T.RESOLVED_ENDPOINT)", Endpoint.class, SdkInternalExecutionAttribute.class); - b.beginControlFlow("if (resolvedEndpoint.headers().isEmpty())"); + b.beginControlFlow("if (resolvedEndpoint == null || $T.isNullOrEmpty(resolvedEndpoint.headers()))", + CollectionUtils.class); b.addStatement("return context.httpRequest()"); b.endControlFlow(); diff --git a/codegen/src/test/resources/software/amazon/awssdk/codegen/poet/rules/endpoint-resolve-interceptor-preSra.java b/codegen/src/test/resources/software/amazon/awssdk/codegen/poet/rules/endpoint-resolve-interceptor-preSra.java index cf27b03ed0c3..e40fb151f74f 100644 --- a/codegen/src/test/resources/software/amazon/awssdk/codegen/poet/rules/endpoint-resolve-interceptor-preSra.java +++ b/codegen/src/test/resources/software/amazon/awssdk/codegen/poet/rules/endpoint-resolve-interceptor-preSra.java @@ -110,7 +110,7 @@ public SdkRequest modifyRequest(Context.ModifyRequest context, ExecutionAttribut @Override public SdkHttpRequest modifyHttpRequest(Context.ModifyHttpRequest context, ExecutionAttributes executionAttributes) { Endpoint resolvedEndpoint = executionAttributes.getAttribute(SdkInternalExecutionAttribute.RESOLVED_ENDPOINT); - if (resolvedEndpoint.headers().isEmpty()) { + if (resolvedEndpoint == null || CollectionUtils.isNullOrEmpty(resolvedEndpoint.headers())) { return context.httpRequest(); } SdkHttpRequest.Builder httpRequestBuilder = context.httpRequest().toBuilder(); diff --git a/codegen/src/test/resources/software/amazon/awssdk/codegen/poet/rules/endpoint-resolve-interceptor-with-endpointsbasedauth.java b/codegen/src/test/resources/software/amazon/awssdk/codegen/poet/rules/endpoint-resolve-interceptor-with-endpointsbasedauth.java index 6d6aa1dc2138..116ad513318d 100644 --- a/codegen/src/test/resources/software/amazon/awssdk/codegen/poet/rules/endpoint-resolve-interceptor-with-endpointsbasedauth.java +++ b/codegen/src/test/resources/software/amazon/awssdk/codegen/poet/rules/endpoint-resolve-interceptor-with-endpointsbasedauth.java @@ -102,7 +102,7 @@ public SdkRequest modifyRequest(Context.ModifyRequest context, ExecutionAttribut @Override public SdkHttpRequest modifyHttpRequest(Context.ModifyHttpRequest context, ExecutionAttributes executionAttributes) { Endpoint resolvedEndpoint = executionAttributes.getAttribute(SdkInternalExecutionAttribute.RESOLVED_ENDPOINT); - if (resolvedEndpoint.headers().isEmpty()) { + if (resolvedEndpoint == null || CollectionUtils.isNullOrEmpty(resolvedEndpoint.headers())) { return context.httpRequest(); } SdkHttpRequest.Builder httpRequestBuilder = context.httpRequest().toBuilder(); diff --git a/codegen/src/test/resources/software/amazon/awssdk/codegen/poet/rules/endpoint-resolve-interceptor-with-multiauthsigv4a.java b/codegen/src/test/resources/software/amazon/awssdk/codegen/poet/rules/endpoint-resolve-interceptor-with-multiauthsigv4a.java index 858d39fc7a69..dd81bd471965 100644 --- a/codegen/src/test/resources/software/amazon/awssdk/codegen/poet/rules/endpoint-resolve-interceptor-with-multiauthsigv4a.java +++ b/codegen/src/test/resources/software/amazon/awssdk/codegen/poet/rules/endpoint-resolve-interceptor-with-multiauthsigv4a.java @@ -91,7 +91,7 @@ public SdkRequest modifyRequest(Context.ModifyRequest context, ExecutionAttribut @Override public SdkHttpRequest modifyHttpRequest(Context.ModifyHttpRequest context, ExecutionAttributes executionAttributes) { Endpoint resolvedEndpoint = executionAttributes.getAttribute(SdkInternalExecutionAttribute.RESOLVED_ENDPOINT); - if (resolvedEndpoint.headers().isEmpty()) { + if (resolvedEndpoint == null || CollectionUtils.isNullOrEmpty(resolvedEndpoint.headers())) { return context.httpRequest(); } SdkHttpRequest.Builder httpRequestBuilder = context.httpRequest().toBuilder(); diff --git a/codegen/src/test/resources/software/amazon/awssdk/codegen/poet/rules/endpoint-resolve-interceptor.java b/codegen/src/test/resources/software/amazon/awssdk/codegen/poet/rules/endpoint-resolve-interceptor.java index abdfedab0455..547cbaca685f 100644 --- a/codegen/src/test/resources/software/amazon/awssdk/codegen/poet/rules/endpoint-resolve-interceptor.java +++ b/codegen/src/test/resources/software/amazon/awssdk/codegen/poet/rules/endpoint-resolve-interceptor.java @@ -93,7 +93,7 @@ public SdkRequest modifyRequest(Context.ModifyRequest context, ExecutionAttribut @Override public SdkHttpRequest modifyHttpRequest(Context.ModifyHttpRequest context, ExecutionAttributes executionAttributes) { Endpoint resolvedEndpoint = executionAttributes.getAttribute(SdkInternalExecutionAttribute.RESOLVED_ENDPOINT); - if (resolvedEndpoint.headers().isEmpty()) { + if (resolvedEndpoint == null || CollectionUtils.isNullOrEmpty(resolvedEndpoint.headers())) { return context.httpRequest(); } SdkHttpRequest.Builder httpRequestBuilder = context.httpRequest().toBuilder(); diff --git a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/presignedurl/DefaultAsyncPresignedUrlManager.java b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/presignedurl/DefaultAsyncPresignedUrlManager.java index 75c4e581befb..0e757581e25b 100644 --- a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/presignedurl/DefaultAsyncPresignedUrlManager.java +++ b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/presignedurl/DefaultAsyncPresignedUrlManager.java @@ -98,7 +98,7 @@ public CompletableFuture getObject( try { apiCallMetricCollector.reportMetric(CoreMetric.SERVICE_ID, "S3"); //TODO: Discuss if we need to change OPERATION_NAME as part of Surface API Review - apiCallMetricCollector.reportMetric(CoreMetric.OPERATION_NAME, "GetObject"); + apiCallMetricCollector.reportMetric(CoreMetric.OPERATION_NAME, "PresignedUrlGetObject"); Pair, CompletableFuture> pair = AsyncResponseTransformerUtils.wrapWithEndOfStreamFuture(asyncResponseTransformer); @@ -113,7 +113,7 @@ public CompletableFuture getObject( CompletableFuture executeFuture = clientHandler.execute( new ClientExecutionParams() - .withOperationName("GetObject") + .withOperationName("PresignedUrlGetObject") .withProtocolMetadata(protocolMetadata) .withResponseHandler(responseHandler) .withErrorResponseHandler(errorResponseHandler) diff --git a/services/s3/src/main/java/software/amazon/awssdk/services/s3/presignedurl/AsyncPresignedUrlManager.java b/services/s3/src/main/java/software/amazon/awssdk/services/s3/presignedurl/AsyncPresignedUrlManager.java index 6342213a679a..93333a684ff5 100644 --- a/services/s3/src/main/java/software/amazon/awssdk/services/s3/presignedurl/AsyncPresignedUrlManager.java +++ b/services/s3/src/main/java/software/amazon/awssdk/services/s3/presignedurl/AsyncPresignedUrlManager.java @@ -15,14 +15,14 @@ package software.amazon.awssdk.services.s3.presignedurl; +import java.nio.file.Path; import java.util.concurrent.CompletableFuture; +import java.util.function.Consumer; import software.amazon.awssdk.annotations.SdkPublicApi; import software.amazon.awssdk.core.async.AsyncResponseTransformer; import software.amazon.awssdk.core.exception.SdkClientException; import software.amazon.awssdk.services.s3.S3AsyncClient; import software.amazon.awssdk.services.s3.model.GetObjectResponse; -import software.amazon.awssdk.services.s3.model.InvalidObjectStateException; -import software.amazon.awssdk.services.s3.model.NoSuchKeyException; import software.amazon.awssdk.services.s3.model.S3Exception; import software.amazon.awssdk.services.s3.presignedurl.model.PresignedUrlGetObjectRequest; @@ -36,46 +36,89 @@ public interface AsyncPresignedUrlManager { *

* Downloads an S3 object asynchronously using a presigned URL. *

+ * *

- * This operation uses a presigned URL that contains all necessary authentication information, eliminating the - * need for AWS credentials at request time. The presigned URL must be valid and not expired. + * This operation uses a presigned URL to download an object from Amazon S3. The presigned URL must be valid and not expired. *

- *
- *
Range Requests
- *
+ * *

- * Supports partial object downloads using HTTP Range headers. Specify the range parameter - * in the request to download only a portion of the object (e.g., "bytes=0-1023"). + * To download a specific byte range of the object, use the range parameter in the request. *

- *
- *
* - * @param request The presigned URL request containing the URL and optional range parameters - * @param responseTransformer Transforms the response to the desired return type. See - * {@link software.amazon.awssdk.core.async.AsyncResponseTransformer} for pre-built - * implementations like downloading to a file or converting to bytes. + * @param request The presigned URL request containing the URL and optional parameters + * @param responseTransformer Transforms the response to the desired return type * @param The type of the transformed response - * @return A {@link CompletableFuture} containing the transformed result of the AsyncResponseTransformer - * @throws software.amazon.awssdk.services.s3.model.NoSuchKeyException The specified object does not exist - * @throws software.amazon.awssdk.services.s3.model.InvalidObjectStateException Object is archived and must be restored before - * retrieval - * @throws software.amazon.awssdk.core.exception.SdkClientException If any client side error occurs such as - * network failures or invalid presigned URL - * @throws S3Exception Base class for all S3 service exceptions. - * Unknown exceptions will be thrown as an - * instance of this type. + * @return A {@link CompletableFuture} containing the transformed result + * @throws SdkClientException If any client side error occurs + * @throws S3Exception Base class for all S3 service exceptions */ default CompletableFuture getObject(PresignedUrlGetObjectRequest request, - AsyncResponseTransformer responseTransformer) throws NoSuchKeyException, - InvalidObjectStateException, - SdkClientException, - S3Exception { + AsyncResponseTransformer responseTransformer) { throw new UnsupportedOperationException(); } - // TODO: Add convenience methods : - // - getObject(Consumer, AsyncResponseTransformer) - consumer-based request building - // - getObject(PresignedUrlGetObjectRequest, Path) - direct file download - // - getObject(Consumer, Path) - consumer + file download -} + /** + *

+ * Downloads an S3 object asynchronously using a presigned URL with a consumer-based request builder. + *

+ * + *

+ * This is a convenience method that creates a {@link PresignedUrlGetObjectRequest} using the provided consumer. + *

+ * + * @param requestConsumer Consumer that will configure a {@link PresignedUrlGetObjectRequest.Builder} + * @param responseTransformer Transforms the response to the desired return type + * @param The type of the transformed response + * @return A {@link CompletableFuture} containing the transformed result + * @throws SdkClientException If any client side error occurs + * @throws S3Exception Base class for all S3 service exceptions + */ + default CompletableFuture getObject(Consumer requestConsumer, + AsyncResponseTransformer responseTransformer) { + return getObject(PresignedUrlGetObjectRequest.builder().applyMutation(requestConsumer).build(), responseTransformer); + } + + /** + *

+ * Downloads an S3 object asynchronously using a presigned URL and saves it to the specified file path. + *

+ * + *

+ * This is a convenience method that uses {@link AsyncResponseTransformer#toFile(Path)} to save the object + * directly to a file. + *

+ * + * @param request The presigned URL request containing the URL and optional parameters + * @param destinationPath The path where the downloaded object will be saved + * @return A {@link CompletableFuture} containing the {@link GetObjectResponse} + * @throws SdkClientException If any client side error occurs + * @throws S3Exception Base class for all S3 service exceptions + */ + default CompletableFuture getObject(PresignedUrlGetObjectRequest request, + Path destinationPath) { + return getObject(request, AsyncResponseTransformer.toFile(destinationPath)); + } + + /** + *

+ * Downloads an S3 object asynchronously using a presigned URL with a consumer-based request builder + * and saves it to the specified file path. + *

+ * + *

+ * This is a convenience method that combines consumer-based request building with file-based response handling. + *

+ * + * @param requestConsumer Consumer that will configure a {@link PresignedUrlGetObjectRequest.Builder} + * @param destinationPath The path where the downloaded object will be saved + * @return A {@link CompletableFuture} containing the {@link GetObjectResponse} + * @throws SdkClientException If any client side error occurs + * @throws S3Exception Base class for all S3 service exceptions + */ + default CompletableFuture getObject(Consumer requestConsumer, + Path destinationPath) { + return getObject(requestConsumer, AsyncResponseTransformer.toFile(destinationPath)); + } +} \ No newline at end of file diff --git a/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/presignedurl/DefaultAsyncPresignedUrlManagerTest.java b/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/presignedurl/DefaultAsyncPresignedUrlManagerTest.java index b74ba0089189..e569287aa582 100644 --- a/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/presignedurl/DefaultAsyncPresignedUrlManagerTest.java +++ b/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/presignedurl/DefaultAsyncPresignedUrlManagerTest.java @@ -256,7 +256,7 @@ void given_AsyncPresignedUrlManager_when_ExecutingDifferentScenarios_then_Should verify(mockPublisher).publish(metricsCaptor.capture()); MetricCollection capturedMetrics = metricsCaptor.getValue(); assertThat(capturedMetrics.metricValues(CoreMetric.SERVICE_ID)).contains("S3"); - assertThat(capturedMetrics.metricValues(CoreMetric.OPERATION_NAME)).contains("GetObject"); + assertThat(capturedMetrics.metricValues(CoreMetric.OPERATION_NAME)).contains("PresignedUrlGetObject"); break; } } diff --git a/services/s3/src/test/java/software/amazon/awssdk/services/s3/presignedurl/AsyncPresignedUrlManagerTest.java b/services/s3/src/test/java/software/amazon/awssdk/services/s3/presignedurl/AsyncPresignedUrlManagerTest.java new file mode 100644 index 000000000000..3441b46d142c --- /dev/null +++ b/services/s3/src/test/java/software/amazon/awssdk/services/s3/presignedurl/AsyncPresignedUrlManagerTest.java @@ -0,0 +1,522 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.services.s3.presignedurl; + +import static com.github.tomakehurst.wiremock.client.WireMock.aResponse; +import static com.github.tomakehurst.wiremock.client.WireMock.exactly; +import static com.github.tomakehurst.wiremock.client.WireMock.get; +import static com.github.tomakehurst.wiremock.client.WireMock.getRequestedFor; +import static com.github.tomakehurst.wiremock.client.WireMock.stubFor; +import static com.github.tomakehurst.wiremock.client.WireMock.urlEqualTo; +import static com.github.tomakehurst.wiremock.client.WireMock.verify; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import java.net.URI; +import java.net.URL; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; +import java.util.Random; +import java.util.UUID; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; + +import com.github.tomakehurst.wiremock.http.Fault; +import com.github.tomakehurst.wiremock.junit5.WireMockRuntimeInfo; +import com.github.tomakehurst.wiremock.junit5.WireMockTest; +import com.github.tomakehurst.wiremock.stubbing.Scenario; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import software.amazon.awssdk.core.ResponseBytes; +import software.amazon.awssdk.core.async.AsyncResponseTransformer; +import software.amazon.awssdk.core.exception.SdkClientException; +import software.amazon.awssdk.services.s3.S3AsyncClient; +import software.amazon.awssdk.services.s3.model.GetObjectResponse; +import software.amazon.awssdk.services.s3.presignedurl.model.PresignedUrlGetObjectRequest; + +@WireMockTest +public class AsyncPresignedUrlManagerTest { + + private S3AsyncClient s3AsyncClient; + private AsyncPresignedUrlManager presignedUrlManager; + + @TempDir + Path tempDir; + + @BeforeEach + void setup(WireMockRuntimeInfo wireMockRuntimeInfo) { + s3AsyncClient = S3AsyncClient.builder() + .endpointOverride(URI.create(wireMockRuntimeInfo.getHttpBaseUrl())) + .build(); + presignedUrlManager = s3AsyncClient.presignedUrlManager(); + } + + @AfterEach + void cleanup() { + if (s3AsyncClient != null) { + s3AsyncClient.close(); + } + } + + @Nested + class BasicFunctionality { + @Test + void givenS3AsyncClient_whenPresignedUrlManagerRequested_thenReturnsNonNullInstance() { + assertThat(presignedUrlManager).isNotNull(); + } + + @Test + void givenValidPresignedUrl_whenGetObjectCalled_thenReturnsExpectedContent(WireMockRuntimeInfo wireMockRuntimeInfo) throws Exception { + String testContent = "Hello world"; + String testETag = "\"d6eb32081c822ed572b70567826d9d9d\""; + String presignedUrlPath = "/presigned-test-object"; + + stubSuccessResponse(presignedUrlPath, testContent, testETag); + + URL presignedUrl = createPresignedUrl(wireMockRuntimeInfo, presignedUrlPath); + PresignedUrlGetObjectRequest request = createRequest(presignedUrl); + + CompletableFuture> result = + presignedUrlManager.getObject(request, AsyncResponseTransformer.toBytes()); + + ResponseBytes response = result.get(); + assertSuccessfulResponse(response, testContent, testETag); + } + + @Test + void givenValidPresignedUrl_whenGetObjectWithConsumerBuilderCalled_thenReturnsExpectedContent(WireMockRuntimeInfo wireMockRuntimeInfo) throws Exception { + String testContent = "Hello consumer"; + String testETag = "\"c1234567890abcdef1234567890abcdef\""; + String presignedUrlPath = "/presigned-test-consumer"; + + stubSuccessResponse(presignedUrlPath, testContent, testETag); + + URL presignedUrl = createPresignedUrl(wireMockRuntimeInfo, presignedUrlPath); + + CompletableFuture> result = + presignedUrlManager.getObject( + request -> request.presignedUrl(presignedUrl), + AsyncResponseTransformer.toBytes() + ); + + ResponseBytes response = result.get(); + assertSuccessfulResponse(response, testContent, testETag); + } + + @Test + void givenEmptyFile_whenGetObjectCalled_thenReturnsEmptyContent(WireMockRuntimeInfo wireMockRuntimeInfo) throws Exception { + String testContent = ""; + String testETag = "\"empty-file-etag\""; + String presignedUrlPath = "/empty-file-memory"; + + // Using custom stubbing for empty file to include Content-Length header + stubFor(get(urlEqualTo(presignedUrlPath)) + .willReturn(aResponse() + .withStatus(200) + .withHeader("ETag", testETag) + .withHeader("Content-Length", "0") + .withBody(testContent))); + + URL presignedUrl = createPresignedUrl(wireMockRuntimeInfo, presignedUrlPath); + PresignedUrlGetObjectRequest request = createRequest(presignedUrl); + + CompletableFuture> result = + presignedUrlManager.getObject(request, AsyncResponseTransformer.toBytes()); + + ResponseBytes response = result.get(); + assertThat(response).isNotNull(); + assertThat(response.asByteArray()).isEmpty(); + assertThat(response.response().eTag()).isEqualTo(testETag); + } + + @Test + void givenRangeRequest_whenGetObjectCalled_thenReturnsPartialContent(WireMockRuntimeInfo wireMockRuntimeInfo) throws Exception { + String expectedContent = "Hello"; + String contentRange = "bytes 0-4/11"; + String range = "bytes=0-4"; + String testETag = "\"d6eb32081c822ed572b70567826d9d9d\""; + String presignedUrlPath = "/presigned-test-range"; + + // Custom stubbing for range request with 206 status + stubFor(get(urlEqualTo(presignedUrlPath)) + .willReturn(aResponse() + .withStatus(206) + .withHeader("ETag", testETag) + .withHeader("Content-Range", contentRange) + .withBody(expectedContent))); + + URL presignedUrl = createPresignedUrl(wireMockRuntimeInfo, presignedUrlPath); + PresignedUrlGetObjectRequest request = PresignedUrlGetObjectRequest.builder() + .presignedUrl(presignedUrl) + .range(range) + .build(); + + CompletableFuture> result = + presignedUrlManager.getObject(request, AsyncResponseTransformer.toBytes()); + + ResponseBytes response = result.get(); + assertThat(response).isNotNull(); + assertThat(response.asUtf8String()).isEqualTo(expectedContent); + } + } + + @Nested + class FileDownloads { + @Test + void givenValidPresignedUrl_whenGetObjectToFileCalled_thenDownloadsFileWithExpectedContent(WireMockRuntimeInfo wireMockRuntimeInfo) throws Exception { + String testContent = "File content for download"; + String testETag = generateRandomETag(); + String presignedUrlPath = "/presigned-test-file"; + + stubSuccessResponse(presignedUrlPath, testContent, testETag); + + URL presignedUrl = createPresignedUrl(wireMockRuntimeInfo, presignedUrlPath); + PresignedUrlGetObjectRequest request = createRequest(presignedUrl); + + Path downloadFile = tempDir.resolve("download-" + UUID.randomUUID() + ".txt"); + + CompletableFuture result = + presignedUrlManager.getObject(request, downloadFile); + + GetObjectResponse response = result.get(); + assertThat(response).isNotNull(); + assertThat(response.eTag()).isEqualTo(testETag); + + String fileContent = new String(Files.readAllBytes(downloadFile), StandardCharsets.UTF_8); + assertThat(fileContent).isEqualTo(testContent); + } + + @Test + void givenValidPresignedUrl_whenGetObjectToFileWithConsumerBuilderCalled_thenDownloadsFileWithExpectedContent(WireMockRuntimeInfo wireMockRuntimeInfo) throws Exception { + String testContent = "File content for consumer download"; + String testETag = generateRandomETag(); + String presignedUrlPath = "/presigned-test-consumer-file"; + + stubSuccessResponse(presignedUrlPath, testContent, testETag); + + URL presignedUrl = createPresignedUrl(wireMockRuntimeInfo, presignedUrlPath); + Path downloadFile = tempDir.resolve("consumer-download-" + UUID.randomUUID() + ".txt"); + + CompletableFuture result = + presignedUrlManager.getObject( + request -> request.presignedUrl(presignedUrl), + downloadFile + ); + + GetObjectResponse response = result.get(); + assertThat(response).isNotNull(); + assertThat(response.eTag()).isEqualTo(testETag); + + String fileContent = new String(Files.readAllBytes(downloadFile), StandardCharsets.UTF_8); + assertThat(fileContent).isEqualTo(testContent); + } + + @Test + void givenEmptyFile_whenGetObjectToFileCalled_thenDownloadsEmptyFile(WireMockRuntimeInfo wireMockRuntimeInfo) throws Exception { + String testContent = ""; + String testETag = "\"empty-file-etag\""; + String presignedUrlPath = "/empty-file"; + + // Custom stubbing for empty file to include Content-Length header + stubFor(get(urlEqualTo(presignedUrlPath)) + .willReturn(aResponse() + .withStatus(200) + .withHeader("ETag", testETag) + .withHeader("Content-Length", "0") + .withBody(testContent))); + + URL presignedUrl = createPresignedUrl(wireMockRuntimeInfo, presignedUrlPath); + PresignedUrlGetObjectRequest request = createRequest(presignedUrl); + + Path downloadFile = tempDir.resolve("empty-file-" + UUID.randomUUID() + ".txt"); + CompletableFuture result = presignedUrlManager.getObject(request, downloadFile); + + GetObjectResponse response = result.get(); + assertThat(response).isNotNull(); + assertThat(response.eTag()).isEqualTo(testETag); + assertThat(Files.size(downloadFile)).isEqualTo(0); + } + + @Test + void givenLargeFile_whenGetObjectCalled_thenDownloadsCompleteContent(WireMockRuntimeInfo wireMockRuntimeInfo) throws Exception { + int contentSize = 1024 * 1024 + 512 * 1024; // 1.5MB + byte[] largeContent = new byte[contentSize]; + new Random().nextBytes(largeContent); + + String testETag = "\"large-file-etag\""; + String presignedUrlPath = "/large-file"; + + // Custom stubbing for large file with Content-Length header + stubFor(get(urlEqualTo(presignedUrlPath)) + .willReturn(aResponse() + .withStatus(200) + .withHeader("ETag", testETag) + .withHeader("Content-Length", String.valueOf(contentSize)) + .withBody(largeContent))); + + URL presignedUrl = createPresignedUrl(wireMockRuntimeInfo, presignedUrlPath); + PresignedUrlGetObjectRequest request = createRequest(presignedUrl); + + Path downloadFile = tempDir.resolve("large-file-" + UUID.randomUUID() + ".bin"); + CompletableFuture result = presignedUrlManager.getObject(request, downloadFile); + + GetObjectResponse response = result.get(); + assertThat(response).isNotNull(); + assertThat(response.eTag()).isEqualTo(testETag); + + assertThat(Files.size(downloadFile)).isEqualTo(contentSize); + + byte[] downloadedContent = Files.readAllBytes(downloadFile); + assertThat(downloadedContent).isEqualTo(largeContent); + } + } + + @Nested + class RetryBehavior { + @Test + void givenNetworkError_whenGetObjectCalled_thenRetriesAndEventuallySucceeds(WireMockRuntimeInfo wireMockRuntimeInfo) throws Exception { + String testContent = "Success after network error"; + String testETag = "\"network-retry-etag\""; + String presignedUrlPath = "/network-error-retry"; + String scenarioName = "network-error-scenario"; + + stubFor(get(urlEqualTo(presignedUrlPath)) + .inScenario(scenarioName) + .whenScenarioStateIs(Scenario.STARTED) + .willReturn(aResponse() + .withFault(Fault.CONNECTION_RESET_BY_PEER)) + .willSetStateTo("after-network-error")); + + stubFor(get(urlEqualTo(presignedUrlPath)) + .inScenario(scenarioName) + .whenScenarioStateIs("after-network-error") + .willReturn(aResponse() + .withStatus(200) + .withHeader("ETag", testETag) + .withBody(testContent))); + + URL presignedUrl = createPresignedUrl(wireMockRuntimeInfo, presignedUrlPath); + PresignedUrlGetObjectRequest request = createRequest(presignedUrl); + + CompletableFuture> result = + presignedUrlManager.getObject(request, AsyncResponseTransformer.toBytes()); + + ResponseBytes response = result.get(); + assertSuccessfulResponse(response, testContent, testETag); + + verify(exactly(2), getRequestedFor(urlEqualTo(presignedUrlPath))); + } + + @Test + void givenTemporaryFailure_whenGetObjectCalled_thenRetriesAndSucceeds(WireMockRuntimeInfo wireMockRuntimeInfo) throws Exception { + String testContent = "Success after retry"; + String testETag = "\"retry-success-etag\""; + String presignedUrlPath = "/retry-test"; + String scenarioName = "retry-scenario"; + + stubFor(get(urlEqualTo(presignedUrlPath)) + .inScenario(scenarioName) + .whenScenarioStateIs(Scenario.STARTED) + .willReturn(aResponse() + .withStatus(503) + .withBody("Service Unavailable")) + .willSetStateTo("retry")); + + stubFor(get(urlEqualTo(presignedUrlPath)) + .inScenario(scenarioName) + .whenScenarioStateIs("retry") + .willReturn(aResponse() + .withStatus(200) + .withHeader("ETag", testETag) + .withBody(testContent))); + + URL presignedUrl = createPresignedUrl(wireMockRuntimeInfo, presignedUrlPath); + PresignedUrlGetObjectRequest request = createRequest(presignedUrl); + + CompletableFuture> result = + presignedUrlManager.getObject(request, AsyncResponseTransformer.toBytes()); + + ResponseBytes response = result.get(); + assertSuccessfulResponse(response, testContent, testETag); + + verify(exactly(2), getRequestedFor(urlEqualTo(presignedUrlPath))); + } + } + + @Nested + class ErrorScenarios { + @Test + void givenMalformedResponse_whenGetObjectCalled_thenCompletableFutureCompletesExceptionally(WireMockRuntimeInfo wireMockRuntimeInfo) throws Exception { + String presignedUrlPath = "/malformed-response"; + stubFor(get(urlEqualTo(presignedUrlPath)) + .willReturn(aResponse() + .withFault(Fault.CONNECTION_RESET_BY_PEER))); + + URL presignedUrl = createPresignedUrl(wireMockRuntimeInfo, presignedUrlPath); + PresignedUrlGetObjectRequest request = createRequest(presignedUrl); + + CompletableFuture> result = + presignedUrlManager.getObject(request, AsyncResponseTransformer.toBytes()); + try { + result.get(); + throw new AssertionError("Expected exception was not thrown"); + } catch (ExecutionException e) { + assertThat(e.getCause()).isInstanceOf(SdkClientException.class); + } + } + + @Test + void givenNotFoundError_whenGetObjectCalled_thenCompletableFutureCompletesExceptionally(WireMockRuntimeInfo wireMockRuntimeInfo) { + String urlPath = "/nonexistent-key"; + + stubFor(get(urlEqualTo(urlPath)) + .willReturn(aResponse() + .withStatus(404) + .withBody("NoSuchKey"))); + + URL presignedUrl; + try { + presignedUrl = new URL(wireMockRuntimeInfo.getHttpBaseUrl() + urlPath); + } catch (Exception e) { + throw new RuntimeException(e); + } + + PresignedUrlGetObjectRequest request = PresignedUrlGetObjectRequest.builder() + .presignedUrl(presignedUrl) + .build(); + + CompletableFuture> result = + presignedUrlManager.getObject(request, AsyncResponseTransformer.toBytes()); + + assertThatThrownBy(() -> result.get()) + .isInstanceOf(ExecutionException.class); + } + + @Test + void givenAccessDeniedError_whenGetObjectCalled_thenCompletableFutureCompletesExceptionally(WireMockRuntimeInfo wireMockRuntimeInfo) { + String urlPath = "/forbidden-key"; + + stubFor(get(urlEqualTo(urlPath)) + .willReturn(aResponse() + .withStatus(403) + .withBody("AccessDenied"))); + + URL presignedUrl; + try { + presignedUrl = new URL(wireMockRuntimeInfo.getHttpBaseUrl() + urlPath); + } catch (Exception e) { + throw new RuntimeException(e); + } + + PresignedUrlGetObjectRequest request = PresignedUrlGetObjectRequest.builder() + .presignedUrl(presignedUrl) + .build(); + + CompletableFuture> result = + presignedUrlManager.getObject(request, AsyncResponseTransformer.toBytes()); + + assertThatThrownBy(() -> result.get()) + .isInstanceOf(ExecutionException.class); + } + } + + @Nested + class ConcurrentOperations { + @Test + void givenMultipleRequests_whenGetObjectCalledConcurrently_thenAllDownloadsComplete(WireMockRuntimeInfo wireMockRuntimeInfo) throws Exception { + int concurrentRequests = 5; + List contents = new ArrayList<>(); + List eTags = new ArrayList<>(); + List paths = new ArrayList<>(); + + for (int i = 0; i < concurrentRequests; i++) { + String content = "Content for concurrent download " + i; + String eTag = "\"concurrent-etag-" + i + "\""; + String path = "/concurrent-" + i; + + contents.add(content); + eTags.add(eTag); + paths.add(path); + + stubSuccessResponse(path, content, eTag); + } + + List>> futures = new ArrayList<>(); + for (int i = 0; i < concurrentRequests; i++) { + URL presignedUrl = createPresignedUrl(wireMockRuntimeInfo, paths.get(i)); + PresignedUrlGetObjectRequest request = createRequest(presignedUrl); + + futures.add(presignedUrlManager.getObject(request, AsyncResponseTransformer.toBytes())); + } + + CompletableFuture allFutures = CompletableFuture.allOf(futures.toArray(new CompletableFuture[0])); + allFutures.get(); + + for (int i = 0; i < concurrentRequests; i++) { + ResponseBytes response = futures.get(i).get(); + assertSuccessfulResponse(response, contents.get(i), eTags.get(i)); + } + } + } + + // Helper methods + private URL createPresignedUrl(WireMockRuntimeInfo wireMockRuntimeInfo, String path) { + try { + return new URL(wireMockRuntimeInfo.getHttpBaseUrl() + path); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + private PresignedUrlGetObjectRequest createRequest(URL presignedUrl) { + return PresignedUrlGetObjectRequest.builder() + .presignedUrl(presignedUrl) + .build(); + } + + private String generateRandomETag() { + return "\"" + UUID.randomUUID().toString().replace("-", "") + "\""; + } + + private void stubSuccessResponse(String path, String content, String eTag) { + stubFor(get(urlEqualTo(path)) + .willReturn(aResponse() + .withStatus(200) + .withHeader("ETag", eTag) + .withBody(content))); + } + + private void stubErrorResponse(String path, int statusCode, String body) { + stubFor(get(urlEqualTo(path)) + .willReturn(aResponse() + .withStatus(statusCode) + .withBody(body))); + } + + private void assertSuccessfulResponse(ResponseBytes response, String expectedContent, String expectedETag) { + assertThat(response).isNotNull(); + assertThat(response.asUtf8String()).isEqualTo(expectedContent); + assertThat(response.response().eTag()).isEqualTo(expectedETag); + } + +} \ No newline at end of file From a2177ef7e2deb0d7c1fae233b5c6ad0b22751d6d Mon Sep 17 00:00:00 2001 From: jencymaryjoseph <35571282+jencymaryjoseph@users.noreply.github.com> Date: Wed, 23 Jul 2025 13:39:36 -0700 Subject: [PATCH 08/30] Rename PresignedUrlManager to PresignedUrlExtension following API review (#6286) * surface api name changes * rename files, remove consumer builder from downloadrequest * some nitpicks * renamed variables --- .../customization/CustomizationConfig.java | 12 ++-- .../awssdk/codegen/poet/PoetExtension.java | 4 +- .../codegen/poet/client/AsyncClientClass.java | 10 +-- .../poet/client/AsyncClientInterface.java | 14 ++-- .../client/DelegatingAsyncClientClass.java | 4 +- .../awssdk/codegen/poet/ClientTestModels.java | 2 +- .../poet/client/AsyncClientClassTest.java | 8 +-- .../poet/client/AsyncClientInterfaceTest.java | 6 +- .../DelegatingAsyncClientClassTest.java | 6 +- .../c2j/presignedurl/customization.config | 2 +- ...tract-async-client-class-presignedurl.java | 8 +-- ...n-async-client-interface-presignedurl.java | 6 +- .../poet/client/test-presignedurl-async.java | 8 +-- .../internal/crt/DefaultS3CrtAsyncClient.java | 7 ++ .../multipart/MultipartS3AsyncClient.java | 7 ++ ...=> DefaultAsyncPresignedUrlExtension.java} | 31 ++++----- ...resignedUrlDownloadRequestMarshaller.java} | 18 ++--- ...> PresignedUrlDownloadRequestWrapper.java} | 20 +++--- ...r.java => AsyncPresignedUrlExtension.java} | 22 +++--- ....java => PresignedUrlDownloadRequest.java} | 35 ++++------ .../codegen-resources/customization.config | 2 +- ...efaultAsyncPresignedUrlExtensionTest.java} | 52 +++++++------- ...gnedUrlDownloadRequestMarshallerTest.java} | 26 +++---- ...esignedUrlDownloadRequestWrapperTest.java} | 12 ++-- ...va => AsyncPresignedUrlExtensionTest.java} | 68 +++++++++---------- ...a => PresignedUrlDownloadRequestTest.java} | 22 +++--- 26 files changed, 207 insertions(+), 205 deletions(-) rename services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/presignedurl/{DefaultAsyncPresignedUrlManager.java => DefaultAsyncPresignedUrlExtension.java} (87%) rename services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/presignedurl/{PresignedUrlGetObjectRequestMarshaller.java => PresignedUrlDownloadRequestMarshaller.java} (78%) rename services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/presignedurl/model/{PresignedUrlGetObjectRequestWrapper.java => PresignedUrlDownloadRequestWrapper.java} (86%) rename services/s3/src/main/java/software/amazon/awssdk/services/s3/presignedurl/{AsyncPresignedUrlManager.java => AsyncPresignedUrlExtension.java} (90%) rename services/s3/src/main/java/software/amazon/awssdk/services/s3/presignedurl/model/{PresignedUrlGetObjectRequest.java => PresignedUrlDownloadRequest.java} (77%) rename services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/presignedurl/{DefaultAsyncPresignedUrlManagerTest.java => DefaultAsyncPresignedUrlExtensionTest.java} (88%) rename services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/presignedurl/{PresignedUrlGetObjectRequestMarshallerTest.java => PresignedUrlDownloadRequestMarshallerTest.java} (88%) rename services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/presignedurl/model/{PresignedUrlGetObjectRequestWrapperTest.java => PresignedUrlDownloadRequestWrapperTest.java} (87%) rename services/s3/src/test/java/software/amazon/awssdk/services/s3/presignedurl/{AsyncPresignedUrlManagerTest.java => AsyncPresignedUrlExtensionTest.java} (89%) rename services/s3/src/test/java/software/amazon/awssdk/services/s3/presignedurl/model/{PresignedUrlGetObjectRequestTest.java => PresignedUrlDownloadRequestTest.java} (83%) diff --git a/codegen/src/main/java/software/amazon/awssdk/codegen/model/config/customization/CustomizationConfig.java b/codegen/src/main/java/software/amazon/awssdk/codegen/model/config/customization/CustomizationConfig.java index 4a7477b40290..805bf02dce20 100644 --- a/codegen/src/main/java/software/amazon/awssdk/codegen/model/config/customization/CustomizationConfig.java +++ b/codegen/src/main/java/software/amazon/awssdk/codegen/model/config/customization/CustomizationConfig.java @@ -352,9 +352,9 @@ public class CustomizationConfig { private boolean batchManagerSupported; /** - * A boolean flag to indicate if Presigned URL Manager is supported. + * A boolean flag to indicate if Presigned URL Extension is supported. */ - private boolean presignedUrlManagerSupported; + private boolean presignedUrlExtensionSupported; /** * A boolean flag to indicate if the fast unmarshaller code path is enabled. @@ -951,12 +951,12 @@ public void setBatchManagerSupported(boolean batchManagerSupported) { this.batchManagerSupported = batchManagerSupported; } - public boolean getPresignedUrlManagerSupported() { - return presignedUrlManagerSupported; + public boolean getPresignedUrlExtensionSupported() { + return presignedUrlExtensionSupported; } - public void setPresignedUrlManagerSupported(boolean presignedUrlManagerSupported) { - this.presignedUrlManagerSupported = presignedUrlManagerSupported; + public void setPresignedUrlExtensionSupported(boolean presignedUrlExtensionSupported) { + this.presignedUrlExtensionSupported = presignedUrlExtensionSupported; } public boolean getEnableFastUnmarshaller() { diff --git a/codegen/src/main/java/software/amazon/awssdk/codegen/poet/PoetExtension.java b/codegen/src/main/java/software/amazon/awssdk/codegen/poet/PoetExtension.java index 54888ce41b57..9e4fea7f6b31 100644 --- a/codegen/src/main/java/software/amazon/awssdk/codegen/poet/PoetExtension.java +++ b/codegen/src/main/java/software/amazon/awssdk/codegen/poet/PoetExtension.java @@ -210,7 +210,7 @@ public ClassName getBatchManagerAsyncInterface() { model.getMetadata().getServiceName() + "AsyncBatchManager"); } - public ClassName getPresignedUrlManagerAsyncInterface() { - return ClassName.get(model.getMetadata().getFullPresignedUrlPackageName(), "AsyncPresignedUrlManager"); + public ClassName getPresignedUrlExtensionAsyncInterface() { + return ClassName.get(model.getMetadata().getFullPresignedUrlPackageName(), "AsyncPresignedUrlExtension"); } } diff --git a/codegen/src/main/java/software/amazon/awssdk/codegen/poet/client/AsyncClientClass.java b/codegen/src/main/java/software/amazon/awssdk/codegen/poet/client/AsyncClientClass.java index d29c3124475b..30014b630453 100644 --- a/codegen/src/main/java/software/amazon/awssdk/codegen/poet/client/AsyncClientClass.java +++ b/codegen/src/main/java/software/amazon/awssdk/codegen/poet/client/AsyncClientClass.java @@ -540,12 +540,12 @@ protected void addBatchManagerMethod(Builder type) { } @Override - protected void addPresignedUrlManagerMethod(Builder type) { - ClassName returnType = poetExtensions.getPresignedUrlManagerAsyncInterface(); + protected void addPresignedUrlExtensionMethod(Builder type) { + ClassName returnType = poetExtensions.getPresignedUrlExtensionAsyncInterface(); String internalPresignedUrlPackage = model.getMetadata().getFullInternalPackageName() + ".presignedurl"; - ClassName implClass = ClassName.get(internalPresignedUrlPackage, "DefaultAsyncPresignedUrlManager"); + ClassName implClass = ClassName.get(internalPresignedUrlPackage, "DefaultAsyncPresignedUrlExtension"); - MethodSpec presignedUrlManager = MethodSpec.methodBuilder("presignedUrlManager") + MethodSpec presignedUrlExtension = MethodSpec.methodBuilder("presignedUrlExtension") .addModifiers(PUBLIC) .addAnnotation(Override.class) .returns(returnType) @@ -556,7 +556,7 @@ protected void addPresignedUrlManagerMethod(Builder type) { implClass) .build(); - type.addMethod(presignedUrlManager); + type.addMethod(presignedUrlExtension); } private MethodSpec resolveMetricPublishersMethod() { diff --git a/codegen/src/main/java/software/amazon/awssdk/codegen/poet/client/AsyncClientInterface.java b/codegen/src/main/java/software/amazon/awssdk/codegen/poet/client/AsyncClientInterface.java index ab4e9223ff40..fe6d40015703 100644 --- a/codegen/src/main/java/software/amazon/awssdk/codegen/poet/client/AsyncClientInterface.java +++ b/codegen/src/main/java/software/amazon/awssdk/codegen/poet/client/AsyncClientInterface.java @@ -99,8 +99,8 @@ public TypeSpec poetSpec() { if (model.getCustomizationConfig().getBatchManagerSupported()) { addBatchManagerMethod(result); } - if (model.getCustomizationConfig().getPresignedUrlManagerSupported()) { - addPresignedUrlManagerMethod(result); + if (model.getCustomizationConfig().getPresignedUrlExtensionSupported()) { + addPresignedUrlExtensionMethod(result); } result.addMethod(serviceClientConfigMethod()); addAdditionalMethods(result); @@ -178,14 +178,14 @@ protected void addBatchManagerMethod(TypeSpec.Builder type) { type.addMethod(batchManagerOperationBody(builder).build()); } - protected void addPresignedUrlManagerMethod(TypeSpec.Builder type) { - ClassName returnType = poetExtensions.getPresignedUrlManagerAsyncInterface(); - MethodSpec.Builder builder = MethodSpec.methodBuilder("presignedUrlManager") + protected void addPresignedUrlExtensionMethod(TypeSpec.Builder type) { + ClassName returnType = poetExtensions.getPresignedUrlExtensionAsyncInterface(); + MethodSpec.Builder builder = MethodSpec.methodBuilder("presignedUrlExtension") .addModifiers(PUBLIC) .returns(returnType) .addJavadoc("Creates an instance of {@link $T} object with the " + "configuration set on this client.", returnType); - type.addMethod(presignedUrlManagerOperationBody(builder).build()); + type.addMethod(presignedUrlExtensionOperationBody(builder).build()); } @Override @@ -564,7 +564,7 @@ protected MethodSpec.Builder batchManagerOperationBody(MethodSpec.Builder builde .addStatement("throw new $T()", UnsupportedOperationException.class); } - protected MethodSpec.Builder presignedUrlManagerOperationBody(MethodSpec.Builder builder) { + protected MethodSpec.Builder presignedUrlExtensionOperationBody(MethodSpec.Builder builder) { return builder.addModifiers(DEFAULT, PUBLIC) .addStatement("throw new $T()", UnsupportedOperationException.class); } diff --git a/codegen/src/main/java/software/amazon/awssdk/codegen/poet/client/DelegatingAsyncClientClass.java b/codegen/src/main/java/software/amazon/awssdk/codegen/poet/client/DelegatingAsyncClientClass.java index be0239e00808..3fd5fcec9f51 100644 --- a/codegen/src/main/java/software/amazon/awssdk/codegen/poet/client/DelegatingAsyncClientClass.java +++ b/codegen/src/main/java/software/amazon/awssdk/codegen/poet/client/DelegatingAsyncClientClass.java @@ -224,7 +224,7 @@ protected MethodSpec.Builder batchManagerOperationBody(MethodSpec.Builder builde } @Override - protected MethodSpec.Builder presignedUrlManagerOperationBody(MethodSpec.Builder builder) { - return builder.addAnnotation(Override.class).addStatement("return delegate.presignedUrlManager()"); + protected MethodSpec.Builder presignedUrlExtensionOperationBody(MethodSpec.Builder builder) { + return builder.addAnnotation(Override.class).addStatement("return delegate.presignedUrlExtension()"); } } diff --git a/codegen/src/test/java/software/amazon/awssdk/codegen/poet/ClientTestModels.java b/codegen/src/test/java/software/amazon/awssdk/codegen/poet/ClientTestModels.java index 82a80e58652b..7426688dd92d 100644 --- a/codegen/src/test/java/software/amazon/awssdk/codegen/poet/ClientTestModels.java +++ b/codegen/src/test/java/software/amazon/awssdk/codegen/poet/ClientTestModels.java @@ -606,7 +606,7 @@ public static IntermediateModel batchManagerModels() { return new IntermediateModelBuilder(models).build(); } - public static IntermediateModel presignedUrlManagerModels() { + public static IntermediateModel presignedUrlExtensionModels() { File serviceModel = new File(ClientTestModels.class.getResource("client/c2j/presignedurl/service-2.json").getFile()); File customizationModel = new File(ClientTestModels.class.getResource("client/c2j/presignedurl/customization.config").getFile()); diff --git a/codegen/src/test/java/software/amazon/awssdk/codegen/poet/client/AsyncClientClassTest.java b/codegen/src/test/java/software/amazon/awssdk/codegen/poet/client/AsyncClientClassTest.java index 7a7c74375cb9..ab3bfeb68c92 100644 --- a/codegen/src/test/java/software/amazon/awssdk/codegen/poet/client/AsyncClientClassTest.java +++ b/codegen/src/test/java/software/amazon/awssdk/codegen/poet/client/AsyncClientClassTest.java @@ -24,7 +24,7 @@ import static software.amazon.awssdk.codegen.poet.ClientTestModels.customPackageModels; import static software.amazon.awssdk.codegen.poet.ClientTestModels.endpointDiscoveryModels; import static software.amazon.awssdk.codegen.poet.ClientTestModels.opsWithSigv4a; -import static software.amazon.awssdk.codegen.poet.ClientTestModels.presignedUrlManagerModels; +import static software.amazon.awssdk.codegen.poet.ClientTestModels.presignedUrlExtensionModels; import static software.amazon.awssdk.codegen.poet.ClientTestModels.serviceWithCustomContextParamsModels; import static software.amazon.awssdk.codegen.poet.ClientTestModels.queryServiceModels; import static software.amazon.awssdk.codegen.poet.ClientTestModels.restJsonServiceModels; @@ -105,9 +105,9 @@ public void asyncClientBatchManager() { } @Test - public void asyncClientPresignedUrlManager() { - ClassSpec asyncClientPresignedUrlManager = createAsyncClientClass(presignedUrlManagerModels()); - assertThat(asyncClientPresignedUrlManager, generatesTo("test-presignedurl-async.java")); + public void asyncClientPresignedUrlExtension() { + ClassSpec asyncClientPresignedUrlExtension = createAsyncClientClass(presignedUrlExtensionModels()); + assertThat(asyncClientPresignedUrlExtension, generatesTo("test-presignedurl-async.java")); } @Test diff --git a/codegen/src/test/java/software/amazon/awssdk/codegen/poet/client/AsyncClientInterfaceTest.java b/codegen/src/test/java/software/amazon/awssdk/codegen/poet/client/AsyncClientInterfaceTest.java index 1a2f18183e8d..aae45702e837 100644 --- a/codegen/src/test/java/software/amazon/awssdk/codegen/poet/client/AsyncClientInterfaceTest.java +++ b/codegen/src/test/java/software/amazon/awssdk/codegen/poet/client/AsyncClientInterfaceTest.java @@ -17,7 +17,7 @@ import static org.hamcrest.MatcherAssert.assertThat; import static software.amazon.awssdk.codegen.poet.ClientTestModels.batchManagerModels; -import static software.amazon.awssdk.codegen.poet.ClientTestModels.presignedUrlManagerModels; +import static software.amazon.awssdk.codegen.poet.ClientTestModels.presignedUrlExtensionModels; import static software.amazon.awssdk.codegen.poet.ClientTestModels.restJsonServiceModels; import static software.amazon.awssdk.codegen.poet.PoetMatchers.generatesTo; @@ -38,8 +38,8 @@ public void asyncClientInterfaceWithBatchManager() { } @Test - public void asyncClientInterfaceWithPresignedUrlManager() { - ClassSpec asyncClientInterface = new AsyncClientInterface(presignedUrlManagerModels()); + public void asyncClientInterfaceWithPresignedUrlExtension() { + ClassSpec asyncClientInterface = new AsyncClientInterface(presignedUrlExtensionModels()); assertThat(asyncClientInterface, generatesTo("test-json-async-client-interface-presignedurl.java")); } } diff --git a/codegen/src/test/java/software/amazon/awssdk/codegen/poet/client/DelegatingAsyncClientClassTest.java b/codegen/src/test/java/software/amazon/awssdk/codegen/poet/client/DelegatingAsyncClientClassTest.java index a77cb67ab727..a9af3d6a8b3b 100644 --- a/codegen/src/test/java/software/amazon/awssdk/codegen/poet/client/DelegatingAsyncClientClassTest.java +++ b/codegen/src/test/java/software/amazon/awssdk/codegen/poet/client/DelegatingAsyncClientClassTest.java @@ -16,7 +16,7 @@ package software.amazon.awssdk.codegen.poet.client; import static org.hamcrest.MatcherAssert.assertThat; -import static software.amazon.awssdk.codegen.poet.ClientTestModels.presignedUrlManagerModels; +import static software.amazon.awssdk.codegen.poet.ClientTestModels.presignedUrlExtensionModels; import static software.amazon.awssdk.codegen.poet.ClientTestModels.restJsonServiceModels; import static software.amazon.awssdk.codegen.poet.PoetMatchers.generatesTo; @@ -31,9 +31,9 @@ public void delegatingAsyncClientClass() { } @Test - public void delegatingAsyncClientClassWithPresignedUrlManager() { + public void delegatingAsyncClientClassWithPresignedUrlExtension() { DelegatingAsyncClientClass asyncClientDecoratorAbstractClass = - new DelegatingAsyncClientClass(presignedUrlManagerModels()); + new DelegatingAsyncClientClass(presignedUrlExtensionModels()); assertThat(asyncClientDecoratorAbstractClass, generatesTo("test-abstract-async-client-class-presignedurl.java")); } } diff --git a/codegen/src/test/resources/software/amazon/awssdk/codegen/poet/client/c2j/presignedurl/customization.config b/codegen/src/test/resources/software/amazon/awssdk/codegen/poet/client/c2j/presignedurl/customization.config index cfc5b2874b7a..17d6c2306501 100644 --- a/codegen/src/test/resources/software/amazon/awssdk/codegen/poet/client/c2j/presignedurl/customization.config +++ b/codegen/src/test/resources/software/amazon/awssdk/codegen/poet/client/c2j/presignedurl/customization.config @@ -1,3 +1,3 @@ { - "presignedUrlManagerSupported": true + "presignedUrlExtensionSupported": true } \ No newline at end of file diff --git a/codegen/src/test/resources/software/amazon/awssdk/codegen/poet/client/test-abstract-async-client-class-presignedurl.java b/codegen/src/test/resources/software/amazon/awssdk/codegen/poet/client/test-abstract-async-client-class-presignedurl.java index 715c56f752a7..7e5984afccf7 100644 --- a/codegen/src/test/resources/software/amazon/awssdk/codegen/poet/client/test-abstract-async-client-class-presignedurl.java +++ b/codegen/src/test/resources/software/amazon/awssdk/codegen/poet/client/test-abstract-async-client-class-presignedurl.java @@ -8,7 +8,7 @@ import software.amazon.awssdk.services.json.model.APostOperationRequest; import software.amazon.awssdk.services.json.model.APostOperationResponse; import software.amazon.awssdk.services.json.model.JsonRequest; -import software.amazon.awssdk.services.json.presignedurl.AsyncPresignedUrlManager; +import software.amazon.awssdk.services.json.presignedurl.AsyncPresignedUrlExtension; import software.amazon.awssdk.utils.Validate; @Generated("software.amazon.awssdk:codegen") @@ -51,11 +51,11 @@ public CompletableFuture aPostOperation(APostOperationRe } /** - * Creates an instance of {@link AsyncPresignedUrlManager} object with the configuration set on this client. + * Creates an instance of {@link AsyncPresignedUrlExtension} object with the configuration set on this client. */ @Override - public AsyncPresignedUrlManager presignedUrlManager() { - return delegate.presignedUrlManager(); + public AsyncPresignedUrlExtension presignedUrlExtension() { + return delegate.presignedUrlExtension(); } @Override diff --git a/codegen/src/test/resources/software/amazon/awssdk/codegen/poet/client/test-json-async-client-interface-presignedurl.java b/codegen/src/test/resources/software/amazon/awssdk/codegen/poet/client/test-json-async-client-interface-presignedurl.java index 631198b323bd..4b5bfc7eb400 100644 --- a/codegen/src/test/resources/software/amazon/awssdk/codegen/poet/client/test-json-async-client-interface-presignedurl.java +++ b/codegen/src/test/resources/software/amazon/awssdk/codegen/poet/client/test-json-async-client-interface-presignedurl.java @@ -8,7 +8,7 @@ import software.amazon.awssdk.awscore.AwsClient; import software.amazon.awssdk.services.json.model.APostOperationRequest; import software.amazon.awssdk.services.json.model.APostOperationResponse; -import software.amazon.awssdk.services.json.presignedurl.AsyncPresignedUrlManager; +import software.amazon.awssdk.services.json.presignedurl.AsyncPresignedUrlExtension; /** * Service client for accessing Json Service asynchronously. This can be created using the static {@link #builder()} @@ -94,9 +94,9 @@ default CompletableFuture aPostOperation(Consumer aPostOperation(APostOperationRe } @Override - public AsyncPresignedUrlManager presignedUrlManager() { - return new DefaultAsyncPresignedUrlManager(clientHandler, protocolFactory, clientConfiguration, protocolMetadata); + public AsyncPresignedUrlExtension presignedUrlExtension() { + return new DefaultAsyncPresignedUrlExtension(clientHandler, protocolFactory, clientConfiguration, protocolMetadata); } @Override diff --git a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/crt/DefaultS3CrtAsyncClient.java b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/crt/DefaultS3CrtAsyncClient.java index e49abb7126e5..eb32823f8deb 100644 --- a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/crt/DefaultS3CrtAsyncClient.java +++ b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/crt/DefaultS3CrtAsyncClient.java @@ -80,6 +80,7 @@ import software.amazon.awssdk.services.s3.model.GetObjectResponse; import software.amazon.awssdk.services.s3.model.PutObjectRequest; import software.amazon.awssdk.services.s3.model.PutObjectResponse; +import software.amazon.awssdk.services.s3.presignedurl.AsyncPresignedUrlExtension; import software.amazon.awssdk.utils.AttributeMap; import software.amazon.awssdk.utils.CollectionUtils; import software.amazon.awssdk.utils.Validate; @@ -501,4 +502,10 @@ private static void validateCrtInClassPath() { + "on the classpath.", e); } } + + @Override + public AsyncPresignedUrlExtension presignedUrlExtension() { + // TODO: Implement presigned URL extension support for CRT client + throw new UnsupportedOperationException("Presigned URL extension is not supported for CRT client"); + } } diff --git a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/MultipartS3AsyncClient.java b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/MultipartS3AsyncClient.java index 499f36a6dd1b..bcb370e8286c 100644 --- a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/MultipartS3AsyncClient.java +++ b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/MultipartS3AsyncClient.java @@ -35,6 +35,7 @@ import software.amazon.awssdk.services.s3.model.PutObjectResponse; import software.amazon.awssdk.services.s3.model.S3Request; import software.amazon.awssdk.services.s3.multipart.MultipartConfiguration; +import software.amazon.awssdk.services.s3.presignedurl.AsyncPresignedUrlExtension; import software.amazon.awssdk.utils.Validate; /** @@ -111,4 +112,10 @@ protected CompletableFuture invokeOperat }; return new MultipartS3AsyncClient(clientWithUserAgent, multipartConfiguration, checksumEnabled); } + + @Override + public AsyncPresignedUrlExtension presignedUrlExtension() { + // TODO: Implement presigned URL extension support for multipart client + throw new UnsupportedOperationException("Presigned URL extension is not supported for multipart client"); + } } diff --git a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/presignedurl/DefaultAsyncPresignedUrlManager.java b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/presignedurl/DefaultAsyncPresignedUrlExtension.java similarity index 87% rename from services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/presignedurl/DefaultAsyncPresignedUrlManager.java rename to services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/presignedurl/DefaultAsyncPresignedUrlExtension.java index 0e757581e25b..db7515b24b54 100644 --- a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/presignedurl/DefaultAsyncPresignedUrlManager.java +++ b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/presignedurl/DefaultAsyncPresignedUrlExtension.java @@ -44,22 +44,22 @@ import software.amazon.awssdk.metrics.NoOpMetricCollector; import software.amazon.awssdk.protocols.xml.AwsS3ProtocolFactory; import software.amazon.awssdk.protocols.xml.XmlOperationMetadata; -import software.amazon.awssdk.services.s3.internal.presignedurl.model.PresignedUrlGetObjectRequestWrapper; +import software.amazon.awssdk.services.s3.internal.presignedurl.model.PresignedUrlDownloadRequestWrapper; import software.amazon.awssdk.services.s3.model.GetObjectResponse; import software.amazon.awssdk.services.s3.model.InvalidObjectStateException; import software.amazon.awssdk.services.s3.model.NoSuchKeyException; import software.amazon.awssdk.services.s3.model.S3Exception; -import software.amazon.awssdk.services.s3.presignedurl.AsyncPresignedUrlManager; -import software.amazon.awssdk.services.s3.presignedurl.model.PresignedUrlGetObjectRequest; +import software.amazon.awssdk.services.s3.presignedurl.AsyncPresignedUrlExtension; +import software.amazon.awssdk.services.s3.presignedurl.model.PresignedUrlDownloadRequest; import software.amazon.awssdk.utils.CompletableFutureUtils; import software.amazon.awssdk.utils.Pair; /** - * Default implementation of {@link AsyncPresignedUrlManager} for executing S3 operations asynchronously using presigned URLs. + * Default implementation of {@link AsyncPresignedUrlExtension} for executing S3 operations asynchronously using presigned URLs. */ @SdkInternalApi -public final class DefaultAsyncPresignedUrlManager implements AsyncPresignedUrlManager { - private static final Logger log = LoggerFactory.getLogger(DefaultAsyncPresignedUrlManager.class); +public final class DefaultAsyncPresignedUrlExtension implements AsyncPresignedUrlExtension { + private static final Logger log = LoggerFactory.getLogger(DefaultAsyncPresignedUrlExtension.class); private final AsyncClientHandler clientHandler; private final AwsS3ProtocolFactory protocolFactory; @@ -67,7 +67,7 @@ public final class DefaultAsyncPresignedUrlManager implements AsyncPresignedUrlM private final List metricPublishers; private final AwsProtocolMetadata protocolMetadata; - public DefaultAsyncPresignedUrlManager(AsyncClientHandler clientHandler, + public DefaultAsyncPresignedUrlExtension(AsyncClientHandler clientHandler, AwsS3ProtocolFactory protocolFactory, SdkClientConfiguration clientConfiguration, AwsProtocolMetadata protocolMetadata) { @@ -82,14 +82,14 @@ public DefaultAsyncPresignedUrlManager(AsyncClientHandler clientHandler, @Override public CompletableFuture getObject( - PresignedUrlGetObjectRequest presignedUrlGetObjectRequest, + PresignedUrlDownloadRequest presignedUrlDownloadRequest, AsyncResponseTransformer asyncResponseTransformer) throws NoSuchKeyException, InvalidObjectStateException, AwsServiceException, SdkClientException, S3Exception { - PresignedUrlGetObjectRequestWrapper internalRequest = PresignedUrlGetObjectRequestWrapper.builder() - .url(presignedUrlGetObjectRequest.presignedUrl()) - .range(presignedUrlGetObjectRequest.range()) + PresignedUrlDownloadRequestWrapper internalRequest = PresignedUrlDownloadRequestWrapper.builder() + .url(presignedUrlDownloadRequest.presignedUrl()) + .range(presignedUrlDownloadRequest.range()) .build(); MetricCollector apiCallMetricCollector = metricPublishers.isEmpty() ? @@ -97,8 +97,7 @@ public CompletableFuture getObject( try { apiCallMetricCollector.reportMetric(CoreMetric.SERVICE_ID, "S3"); - //TODO: Discuss if we need to change OPERATION_NAME as part of Surface API Review - apiCallMetricCollector.reportMetric(CoreMetric.OPERATION_NAME, "PresignedUrlGetObject"); + apiCallMetricCollector.reportMetric(CoreMetric.OPERATION_NAME, "PresignedUrlDownload"); Pair, CompletableFuture> pair = AsyncResponseTransformerUtils.wrapWithEndOfStreamFuture(asyncResponseTransformer); @@ -112,8 +111,8 @@ public CompletableFuture getObject( HttpResponseHandler errorResponseHandler = protocolFactory.createErrorResponseHandler(); CompletableFuture executeFuture = clientHandler.execute( - new ClientExecutionParams() - .withOperationName("PresignedUrlGetObject") + new ClientExecutionParams() + .withOperationName("PresignedUrlDownload") .withProtocolMetadata(protocolMetadata) .withResponseHandler(responseHandler) .withErrorResponseHandler(errorResponseHandler) @@ -122,7 +121,7 @@ public CompletableFuture getObject( .withMetricCollector(apiCallMetricCollector) // TODO: Deprecate IS_DISCOVERED_ENDPOINT, use new SKIP_ENDPOINT_RESOLUTION for better semantics .putExecutionAttribute(SdkInternalExecutionAttribute.IS_DISCOVERED_ENDPOINT, true) - .withMarshaller(new PresignedUrlGetObjectRequestMarshaller(protocolFactory)), + .withMarshaller(new PresignedUrlDownloadRequestMarshaller(protocolFactory)), asyncResponseTransformer); CompletableFuture whenCompleteFuture = executeFuture.whenComplete((r, e) -> { diff --git a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/presignedurl/PresignedUrlGetObjectRequestMarshaller.java b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/presignedurl/PresignedUrlDownloadRequestMarshaller.java similarity index 78% rename from services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/presignedurl/PresignedUrlGetObjectRequestMarshaller.java rename to services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/presignedurl/PresignedUrlDownloadRequestMarshaller.java index 76e1500f2d54..f0aecfce87da 100644 --- a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/presignedurl/PresignedUrlGetObjectRequestMarshaller.java +++ b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/presignedurl/PresignedUrlDownloadRequestMarshaller.java @@ -24,11 +24,11 @@ import software.amazon.awssdk.protocols.core.OperationInfo; import software.amazon.awssdk.protocols.core.ProtocolMarshaller; import software.amazon.awssdk.protocols.xml.AwsXmlProtocolFactory; -import software.amazon.awssdk.services.s3.internal.presignedurl.model.PresignedUrlGetObjectRequestWrapper; +import software.amazon.awssdk.services.s3.internal.presignedurl.model.PresignedUrlDownloadRequestWrapper; import software.amazon.awssdk.utils.Validate; /** - * {@link PresignedUrlGetObjectRequestWrapper} Marshaller + * {@link PresignedUrlDownloadRequestWrapper} Marshaller * *

* Marshalls presigned URL requests by using the complete URL directly and adding optional Range headers. @@ -36,7 +36,7 @@ *

*/ @SdkInternalApi -public class PresignedUrlGetObjectRequestMarshaller implements Marshaller { +public class PresignedUrlDownloadRequestMarshaller implements Marshaller { private static final OperationInfo SDK_OPERATION_BINDING = OperationInfo.builder() .requestUri("").httpMethod(SdkHttpMethod.GET).hasExplicitPayloadMember(false).hasPayloadMembers(false) .putAdditionalMetadata(AwsXmlProtocolFactory.ROOT_MARSHALL_LOCATION_ATTRIBUTE, null) @@ -44,26 +44,26 @@ public class PresignedUrlGetObjectRequestMarshaller implements Marshaller protocolMarshaller = protocolFactory .createProtocolMarshaller(SDK_OPERATION_BINDING); - URI presignedUri = presignedUrlGetObjectRequestWrapper.url().toURI(); + URI presignedUri = presignedUrlDownloadRequestWrapper.url().toURI(); - return protocolMarshaller.marshall(presignedUrlGetObjectRequestWrapper) + return protocolMarshaller.marshall(presignedUrlDownloadRequestWrapper) .toBuilder() .uri(presignedUri) .build(); diff --git a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/presignedurl/model/PresignedUrlGetObjectRequestWrapper.java b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/presignedurl/model/PresignedUrlDownloadRequestWrapper.java similarity index 86% rename from services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/presignedurl/model/PresignedUrlGetObjectRequestWrapper.java rename to services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/presignedurl/model/PresignedUrlDownloadRequestWrapper.java index efd5a9f78167..983c6b9c13eb 100644 --- a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/presignedurl/model/PresignedUrlGetObjectRequestWrapper.java +++ b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/presignedurl/model/PresignedUrlDownloadRequestWrapper.java @@ -37,14 +37,14 @@ * SdkField definitions needed for custom marshalling and is not intended for direct use by SDK users. *

* Note: This is an internal implementation class and should not be used - * directly. Use {@code PresignedUrlGetObjectRequest} for public API interactions. + * directly. Use {@code PresignedUrlDownloadRequest} for public API interactions. */ @SdkInternalApi -public final class PresignedUrlGetObjectRequestWrapper extends S3Request { +public final class PresignedUrlDownloadRequestWrapper extends S3Request { private static final SdkField RANGE_FIELD = SdkField .builder(MarshallingType.STRING) .memberName("Range") - .getter(getter(PresignedUrlGetObjectRequestWrapper::range)) + .getter(getter(PresignedUrlDownloadRequestWrapper::range)) .traits(LocationTrait.builder().location(MarshallLocation.HEADER).locationName("Range") .unmarshallLocationName("Range").build()).build(); @@ -56,7 +56,7 @@ public final class PresignedUrlGetObjectRequestWrapper extends S3Request { private final URL url; private final String range; - private PresignedUrlGetObjectRequestWrapper(Builder builder) { + private PresignedUrlDownloadRequestWrapper(Builder builder) { super(builder); this.url = builder.url; this.range = builder.range; @@ -80,8 +80,8 @@ public Map> sdkFieldNameToField() { return SDK_NAME_TO_FIELD; } - private static Function getter(Function g) { - return obj -> g.apply((PresignedUrlGetObjectRequestWrapper) obj); + private static Function getter(Function g) { + return obj -> g.apply((PresignedUrlDownloadRequestWrapper) obj); } private static Map> memberNameToFieldInitializer() { @@ -110,7 +110,7 @@ public boolean equals(Object obj) { if (!super.equals(obj)) { return false; } - PresignedUrlGetObjectRequestWrapper that = (PresignedUrlGetObjectRequestWrapper) obj; + PresignedUrlDownloadRequestWrapper that = (PresignedUrlDownloadRequestWrapper) obj; return Objects.equals(url, that.url) && Objects.equals(range, that.range); } @@ -129,7 +129,7 @@ public static final class Builder extends S3Request.BuilderImpl { public Builder() { } - Builder(PresignedUrlGetObjectRequestWrapper request) { + Builder(PresignedUrlDownloadRequestWrapper request) { super(request); this.url = request.url(); this.range = request.range(); @@ -146,8 +146,8 @@ public Builder range(String range) { } @Override - public PresignedUrlGetObjectRequestWrapper build() { - return new PresignedUrlGetObjectRequestWrapper(this); + public PresignedUrlDownloadRequestWrapper build() { + return new PresignedUrlDownloadRequestWrapper(this); } } } diff --git a/services/s3/src/main/java/software/amazon/awssdk/services/s3/presignedurl/AsyncPresignedUrlManager.java b/services/s3/src/main/java/software/amazon/awssdk/services/s3/presignedurl/AsyncPresignedUrlExtension.java similarity index 90% rename from services/s3/src/main/java/software/amazon/awssdk/services/s3/presignedurl/AsyncPresignedUrlManager.java rename to services/s3/src/main/java/software/amazon/awssdk/services/s3/presignedurl/AsyncPresignedUrlExtension.java index 93333a684ff5..545eb838c35e 100644 --- a/services/s3/src/main/java/software/amazon/awssdk/services/s3/presignedurl/AsyncPresignedUrlManager.java +++ b/services/s3/src/main/java/software/amazon/awssdk/services/s3/presignedurl/AsyncPresignedUrlExtension.java @@ -24,14 +24,14 @@ import software.amazon.awssdk.services.s3.S3AsyncClient; import software.amazon.awssdk.services.s3.model.GetObjectResponse; import software.amazon.awssdk.services.s3.model.S3Exception; -import software.amazon.awssdk.services.s3.presignedurl.model.PresignedUrlGetObjectRequest; +import software.amazon.awssdk.services.s3.presignedurl.model.PresignedUrlDownloadRequest; /** * Interface for executing S3 operations asynchronously using presigned URLs. This can be accessed using - * {@link S3AsyncClient#presignedUrlManager()}. + * {@link S3AsyncClient#presignedUrlExtension()}. */ @SdkPublicApi -public interface AsyncPresignedUrlManager { +public interface AsyncPresignedUrlExtension { /** *

* Downloads an S3 object asynchronously using a presigned URL. @@ -52,7 +52,7 @@ public interface AsyncPresignedUrlManager { * @throws SdkClientException If any client side error occurs * @throws S3Exception Base class for all S3 service exceptions */ - default CompletableFuture getObject(PresignedUrlGetObjectRequest request, + default CompletableFuture getObject(PresignedUrlDownloadRequest request, AsyncResponseTransformer responseTransformer) { throw new UnsupportedOperationException(); @@ -64,20 +64,20 @@ default CompletableFuture getObject(PresignedUrlGetObjectRequ *

* *

- * This is a convenience method that creates a {@link PresignedUrlGetObjectRequest} using the provided consumer. + * This is a convenience method that creates a {@link PresignedUrlDownloadRequest} using the provided consumer. *

* - * @param requestConsumer Consumer that will configure a {@link PresignedUrlGetObjectRequest.Builder} + * @param requestConsumer Consumer that will configure a {@link PresignedUrlDownloadRequest.Builder} * @param responseTransformer Transforms the response to the desired return type * @param The type of the transformed response * @return A {@link CompletableFuture} containing the transformed result * @throws SdkClientException If any client side error occurs * @throws S3Exception Base class for all S3 service exceptions */ - default CompletableFuture getObject(Consumer requestConsumer, + default CompletableFuture getObject(Consumer requestConsumer, AsyncResponseTransformer responseTransformer) { - return getObject(PresignedUrlGetObjectRequest.builder().applyMutation(requestConsumer).build(), responseTransformer); + return getObject(PresignedUrlDownloadRequest.builder().applyMutation(requestConsumer).build(), responseTransformer); } /** @@ -96,7 +96,7 @@ default CompletableFuture getObject(Consumer getObject(PresignedUrlGetObjectRequest request, + default CompletableFuture getObject(PresignedUrlDownloadRequest request, Path destinationPath) { return getObject(request, AsyncResponseTransformer.toFile(destinationPath)); } @@ -111,13 +111,13 @@ default CompletableFuture getObject(PresignedUrlGetObjectRequ * This is a convenience method that combines consumer-based request building with file-based response handling. *

* - * @param requestConsumer Consumer that will configure a {@link PresignedUrlGetObjectRequest.Builder} + * @param requestConsumer Consumer that will configure a {@link PresignedUrlDownloadRequest.Builder} * @param destinationPath The path where the downloaded object will be saved * @return A {@link CompletableFuture} containing the {@link GetObjectResponse} * @throws SdkClientException If any client side error occurs * @throws S3Exception Base class for all S3 service exceptions */ - default CompletableFuture getObject(Consumer requestConsumer, + default CompletableFuture getObject(Consumer requestConsumer, Path destinationPath) { return getObject(requestConsumer, AsyncResponseTransformer.toFile(destinationPath)); } diff --git a/services/s3/src/main/java/software/amazon/awssdk/services/s3/presignedurl/model/PresignedUrlGetObjectRequest.java b/services/s3/src/main/java/software/amazon/awssdk/services/s3/presignedurl/model/PresignedUrlDownloadRequest.java similarity index 77% rename from services/s3/src/main/java/software/amazon/awssdk/services/s3/presignedurl/model/PresignedUrlGetObjectRequest.java rename to services/s3/src/main/java/software/amazon/awssdk/services/s3/presignedurl/model/PresignedUrlDownloadRequest.java index 20cd9c89d8a2..380c234b82c8 100644 --- a/services/s3/src/main/java/software/amazon/awssdk/services/s3/presignedurl/model/PresignedUrlGetObjectRequest.java +++ b/services/s3/src/main/java/software/amazon/awssdk/services/s3/presignedurl/model/PresignedUrlDownloadRequest.java @@ -17,7 +17,6 @@ import java.net.URL; import java.util.Objects; -import java.util.function.Consumer; import software.amazon.awssdk.annotations.SdkPublicApi; import software.amazon.awssdk.utils.ToString; import software.amazon.awssdk.utils.Validate; @@ -25,15 +24,15 @@ import software.amazon.awssdk.utils.builder.ToCopyableBuilder; /** - * Request object for performing GetObject operations using a presigned URL. + * Request object for performing download operations using a presigned URL. */ @SdkPublicApi -public final class PresignedUrlGetObjectRequest implements ToCopyableBuilder { +public final class PresignedUrlDownloadRequest implements ToCopyableBuilder { private final URL presignedUrl; private final String range; - private PresignedUrlGetObjectRequest(BuilderImpl builder) { + private PresignedUrlDownloadRequest(BuilderImpl builder) { this.presignedUrl = builder.presignedUrl; this.range = builder.range; } @@ -75,16 +74,6 @@ public static Builder builder() { return new BuilderImpl(); } - /** - * Create a {@code PresignedUrlGetObjectRequest} instance with the given configuration - * - * @param builderConsumer the consumer that will configure the builder - * @return a {@code PresignedUrlGetObjectRequest} instance - */ - public static PresignedUrlGetObjectRequest builder(Consumer builderConsumer) { - return builder().applyMutation(builderConsumer).build(); - } - public static Class serializableBuilderClass() { return BuilderImpl.class; } @@ -105,20 +94,20 @@ public boolean equals(Object obj) { if (obj == null || getClass() != obj.getClass()) { return false; } - PresignedUrlGetObjectRequest other = (PresignedUrlGetObjectRequest) obj; + PresignedUrlDownloadRequest other = (PresignedUrlDownloadRequest) obj; return Objects.equals(presignedUrl(), other.presignedUrl()) && Objects.equals(range(), other.range()); } @Override public String toString() { - return ToString.builder("PresignedUrlGetObjectRequest") + return ToString.builder("PresignedUrlDownloadRequest") .add("PresignedUrl", presignedUrl()) .add("Range", range()) .build(); } - public interface Builder extends CopyableBuilder { + public interface Builder extends CopyableBuilder { /** * Sets the presigned URL for the S3 object. * @param presignedUrl @@ -141,9 +130,9 @@ static final class BuilderImpl implements Builder { private BuilderImpl() { } - private BuilderImpl(PresignedUrlGetObjectRequest presignedUrlGetObjectRequest) { - presignedUrl(presignedUrlGetObjectRequest.presignedUrl()); - range(presignedUrlGetObjectRequest.range()); + private BuilderImpl(PresignedUrlDownloadRequest presignedUrlDownloadRequest) { + presignedUrl(presignedUrlDownloadRequest.presignedUrl()); + range(presignedUrlDownloadRequest.range()); } @Override @@ -159,9 +148,9 @@ public Builder range(String range) { } @Override - public PresignedUrlGetObjectRequest build() { + public PresignedUrlDownloadRequest build() { Validate.paramNotNull(presignedUrl, "presignedUrl"); - return new PresignedUrlGetObjectRequest(this); + return new PresignedUrlDownloadRequest(this); } } } diff --git a/services/s3/src/main/resources/codegen-resources/customization.config b/services/s3/src/main/resources/codegen-resources/customization.config index 0d9b8038bd1f..e266868be2e3 100644 --- a/services/s3/src/main/resources/codegen-resources/customization.config +++ b/services/s3/src/main/resources/codegen-resources/customization.config @@ -359,5 +359,5 @@ "className": "software.amazon.awssdk.services.s3.internal.CustomRequestTransformerUtils" } }, - "presignedUrlManagerSupported": true + "presignedUrlExtensionSupported": true } diff --git a/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/presignedurl/DefaultAsyncPresignedUrlManagerTest.java b/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/presignedurl/DefaultAsyncPresignedUrlExtensionTest.java similarity index 88% rename from services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/presignedurl/DefaultAsyncPresignedUrlManagerTest.java rename to services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/presignedurl/DefaultAsyncPresignedUrlExtensionTest.java index e569287aa582..91681fbb8e15 100644 --- a/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/presignedurl/DefaultAsyncPresignedUrlManagerTest.java +++ b/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/presignedurl/DefaultAsyncPresignedUrlExtensionTest.java @@ -67,13 +67,13 @@ import software.amazon.awssdk.services.s3.model.InvalidObjectStateException; import software.amazon.awssdk.services.s3.model.NoSuchKeyException; import software.amazon.awssdk.services.s3.model.S3Exception; -import software.amazon.awssdk.services.s3.presignedurl.model.PresignedUrlGetObjectRequest; +import software.amazon.awssdk.services.s3.presignedurl.model.PresignedUrlDownloadRequest; import software.amazon.awssdk.testutils.service.http.MockAsyncHttpClient; /** - * Tests for {@link DefaultAsyncPresignedUrlManager} using MockAsyncHttpClient to verify HTTP interactions. + * Tests for {@link DefaultAsyncPresignedUrlExtension} using MockAsyncHttpClient to verify HTTP interactions. */ -class DefaultAsyncPresignedUrlManagerTest { +class DefaultAsyncPresignedUrlExtensionTest { private static final String TEST_CONTENT = "test-content"; private static final URI DEFAULT_ENDPOINT = URI.create("https://defaultendpoint.com"); @@ -87,8 +87,8 @@ class DefaultAsyncPresignedUrlManagerTest { "X-Amz-Expires=86400"; private MockAsyncHttpClient mockHttpClient; - private DefaultAsyncPresignedUrlManager presignedUrlManager; - private PresignedUrlGetObjectRequest testRequest; + private DefaultAsyncPresignedUrlExtension presignedUrlExtension; + private PresignedUrlDownloadRequest testRequest; private AwsProtocolMetadata protocolMetadata; private AwsS3ProtocolFactory protocolFactory; private AsyncClientHandler clientHandler; @@ -97,7 +97,7 @@ class DefaultAsyncPresignedUrlManagerTest { void setUp() throws Exception { mockHttpClient = new MockAsyncHttpClient(); URL testPresignedUrl = new URL(TEST_URL); - testRequest = PresignedUrlGetObjectRequest.builder() + testRequest = PresignedUrlDownloadRequest.builder() .presignedUrl(testPresignedUrl) .build(); @@ -108,7 +108,7 @@ void setUp() throws Exception { protocolFactory = initProtocolFactory(clientConfiguration); clientHandler = new AwsAsyncClientHandler(clientConfiguration); - presignedUrlManager = new DefaultAsyncPresignedUrlManager( + presignedUrlExtension = new DefaultAsyncPresignedUrlExtension( clientHandler, protocolFactory, clientConfiguration, protocolMetadata); } @@ -145,13 +145,13 @@ private static Stream requestConfigurationTestCases() { return Stream.of( Arguments.of( "Basic request", - (Consumer) builder -> + (Consumer) builder -> builder.presignedUrl(createTestUrl()), null ), Arguments.of( "Request with range header", - (Consumer) builder -> + (Consumer) builder -> builder.presignedUrl(createTestUrl()).range("bytes=0-1024"), "bytes=0-1024" ) @@ -176,7 +176,7 @@ private static Stream invalidUrlTestCases() { @ParameterizedTest(name = "{0}") @MethodSource("httpResponseTestCases") - void given_AsyncPresignedUrlManager_when_GetObjectWithDifferentHttpResponses_then_ShouldHandleSuccessAndErrorsCorrectly( + void given_AsyncPresignedUrlExtension_when_GetObjectWithDifferentHttpResponses_then_ShouldHandleSuccessAndErrorsCorrectly( String testName, HttpExecuteResponse response, boolean expectSuccess, @@ -188,7 +188,7 @@ void given_AsyncPresignedUrlManager_when_GetObjectWithDifferentHttpResponses_the assertSuccessfulGetObject(testRequest); } else { CompletableFuture> future = - presignedUrlManager.getObject(testRequest, AsyncResponseTransformer.toBytes()); + presignedUrlExtension.getObject(testRequest, AsyncResponseTransformer.toBytes()); assertThatThrownBy(future::join) .hasCauseInstanceOf(expectedExceptionType); @@ -197,19 +197,19 @@ void given_AsyncPresignedUrlManager_when_GetObjectWithDifferentHttpResponses_the @ParameterizedTest(name = "{0}") @MethodSource("requestConfigurationTestCases") - void given_AsyncPresignedUrlManager_when_GetObjectWithDifferentRequestConfigurations_then_ShouldSetCorrectHeaders( + void given_AsyncPresignedUrlExtension_when_GetObjectWithDifferentRequestConfigurations_then_ShouldSetCorrectHeaders( String testName, - Consumer requestCustomizer, + Consumer requestCustomizer, String expectedRangeHeader) throws ExecutionException, InterruptedException { mockHttpClient.stubNextResponse(createSuccessResponse()); - PresignedUrlGetObjectRequest.Builder builder = PresignedUrlGetObjectRequest.builder(); + PresignedUrlDownloadRequest.Builder builder = PresignedUrlDownloadRequest.builder(); requestCustomizer.accept(builder); - PresignedUrlGetObjectRequest request = builder.build(); + PresignedUrlDownloadRequest request = builder.build(); CompletableFuture> future = - presignedUrlManager.getObject(request, AsyncResponseTransformer.toBytes()); + presignedUrlExtension.getObject(request, AsyncResponseTransformer.toBytes()); ResponseBytes result = future.get(); assertThat(result).isNotNull(); @@ -227,14 +227,14 @@ void given_AsyncPresignedUrlManager_when_GetObjectWithDifferentRequestConfigurat @ParameterizedTest(name = "{0}") @MethodSource("additionalTestCases") - void given_AsyncPresignedUrlManager_when_ExecutingDifferentScenarios_then_ShouldBehaveCorrectly( + void given_AsyncPresignedUrlExtension_when_ExecutingDifferentScenarios_then_ShouldBehaveCorrectly( String testName, String testType) throws Exception { switch (testType) { case "CUSTOM_TRANSFORMER": mockHttpClient.stubNextResponse(createSuccessResponse()); CompletableFuture> future = - presignedUrlManager.getObject(testRequest, AsyncResponseTransformer.toBytes()); + presignedUrlExtension.getObject(testRequest, AsyncResponseTransformer.toBytes()); ResponseBytes result = future.get(); assertThat(result.asUtf8String()).isEqualTo(TEST_CONTENT); break; @@ -244,11 +244,11 @@ void given_AsyncPresignedUrlManager_when_ExecutingDifferentScenarios_then_Should SdkClientConfiguration clientConfigWithMetrics = getDefaultSdkConfigs().toBuilder() .option(SdkClientOption.METRIC_PUBLISHERS, Collections.singletonList(mockPublisher)) .build(); - DefaultAsyncPresignedUrlManager managerWithMetrics = new DefaultAsyncPresignedUrlManager( + DefaultAsyncPresignedUrlExtension extensionWithMetrics = new DefaultAsyncPresignedUrlExtension( clientHandler, protocolFactory, clientConfigWithMetrics, protocolMetadata); mockHttpClient.stubNextResponse(createSuccessResponse()); CompletableFuture> metricsFuture = - managerWithMetrics.getObject(testRequest, AsyncResponseTransformer.toBytes()); + extensionWithMetrics.getObject(testRequest, AsyncResponseTransformer.toBytes()); metricsFuture.get(); verify(mockPublisher, atLeastOnce()).publish(any(MetricCollection.class)); @@ -256,21 +256,21 @@ void given_AsyncPresignedUrlManager_when_ExecutingDifferentScenarios_then_Should verify(mockPublisher).publish(metricsCaptor.capture()); MetricCollection capturedMetrics = metricsCaptor.getValue(); assertThat(capturedMetrics.metricValues(CoreMetric.SERVICE_ID)).contains("S3"); - assertThat(capturedMetrics.metricValues(CoreMetric.OPERATION_NAME)).contains("PresignedUrlGetObject"); + assertThat(capturedMetrics.metricValues(CoreMetric.OPERATION_NAME)).contains("PresignedUrlDownload"); break; } } @ParameterizedTest(name = "{0}") @MethodSource("invalidUrlTestCases") - void given_AsyncPresignedUrlManager_when_GetObjectWithInvalidUrl_then_ShouldThrowException( + void given_AsyncPresignedUrlExtension_when_GetObjectWithInvalidUrl_then_ShouldThrowException( String testName, String invalidUrlString) { assertThatThrownBy(() -> { URL invalidUrl = new URL(invalidUrlString); - PresignedUrlGetObjectRequest invalidRequest = PresignedUrlGetObjectRequest.builder() + PresignedUrlDownloadRequest invalidRequest = PresignedUrlDownloadRequest.builder() .presignedUrl(invalidUrl) .build(); - presignedUrlManager.getObject(invalidRequest, AsyncResponseTransformer.toBytes()).join(); + presignedUrlExtension.getObject(invalidRequest, AsyncResponseTransformer.toBytes()).join(); }).satisfiesAnyOf( ex -> assertThat(ex).isInstanceOf(java.net.MalformedURLException.class), ex -> assertThat(ex).isInstanceOf(SdkClientException.class), @@ -324,10 +324,10 @@ private static URL createTestUrl() { } } - private void assertSuccessfulGetObject(PresignedUrlGetObjectRequest request) { + private void assertSuccessfulGetObject(PresignedUrlDownloadRequest request) { try { CompletableFuture> future = - presignedUrlManager.getObject(request, AsyncResponseTransformer.toBytes()); + presignedUrlExtension.getObject(request, AsyncResponseTransformer.toBytes()); ResponseBytes result = future.get(); assertThat(result).isNotNull(); diff --git a/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/presignedurl/PresignedUrlGetObjectRequestMarshallerTest.java b/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/presignedurl/PresignedUrlDownloadRequestMarshallerTest.java similarity index 88% rename from services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/presignedurl/PresignedUrlGetObjectRequestMarshallerTest.java rename to services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/presignedurl/PresignedUrlDownloadRequestMarshallerTest.java index 2311017a29e5..b89794a2a59c 100644 --- a/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/presignedurl/PresignedUrlGetObjectRequestMarshallerTest.java +++ b/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/presignedurl/PresignedUrlDownloadRequestMarshallerTest.java @@ -32,11 +32,11 @@ import software.amazon.awssdk.protocols.core.OperationInfo; import software.amazon.awssdk.protocols.core.ProtocolMarshaller; import software.amazon.awssdk.protocols.xml.AwsXmlProtocolFactory; -import software.amazon.awssdk.services.s3.internal.presignedurl.model.PresignedUrlGetObjectRequestWrapper; +import software.amazon.awssdk.services.s3.internal.presignedurl.model.PresignedUrlDownloadRequestWrapper; -class PresignedUrlGetObjectRequestMarshallerTest { +class PresignedUrlDownloadRequestMarshallerTest { - private PresignedUrlGetObjectRequestMarshaller marshaller; + private PresignedUrlDownloadRequestMarshaller marshaller; private AwsXmlProtocolFactory mockProtocolFactory; private ProtocolMarshaller mockProtocolMarshaller; private URL testUrl; @@ -47,7 +47,7 @@ void setUp() throws Exception { mockProtocolMarshaller = mock(ProtocolMarshaller.class); when(mockProtocolFactory.createProtocolMarshaller(any(OperationInfo.class))) .thenReturn(mockProtocolMarshaller); - marshaller = new PresignedUrlGetObjectRequestMarshaller(mockProtocolFactory); + marshaller = new PresignedUrlDownloadRequestMarshaller(mockProtocolFactory); testUrl = new URL("https://test-bucket.s3.us-east-1.amazonaws.com/test-key?" + "X-Amz-Date=20231215T000000Z&" + @@ -67,10 +67,10 @@ void marshall_withBasicRequest_shouldCreateCorrectHttpRequest() throws Exception .protocol("https") .host("example.com") .build(); - when(mockProtocolMarshaller.marshall(any(PresignedUrlGetObjectRequestWrapper.class))) + when(mockProtocolMarshaller.marshall(any(PresignedUrlDownloadRequestWrapper.class))) .thenReturn(baseRequest); - PresignedUrlGetObjectRequestWrapper request = PresignedUrlGetObjectRequestWrapper.builder() + PresignedUrlDownloadRequestWrapper request = PresignedUrlDownloadRequestWrapper.builder() .url(testUrl) .build(); SdkHttpFullRequest result = marshaller.marshall(request); @@ -114,10 +114,10 @@ void marshall_withValidRangeFormats_shouldAddRangeHeader(String rangeValue) thro .putHeader("Range", rangeValue) // Add the Range header to the mock response .build(); - when(mockProtocolMarshaller.marshall(any(PresignedUrlGetObjectRequestWrapper.class))) + when(mockProtocolMarshaller.marshall(any(PresignedUrlDownloadRequestWrapper.class))) .thenReturn(baseRequest); - PresignedUrlGetObjectRequestWrapper request = PresignedUrlGetObjectRequestWrapper.builder() + PresignedUrlDownloadRequestWrapper request = PresignedUrlDownloadRequestWrapper.builder() .url(testUrl) .range(rangeValue) .build(); @@ -139,10 +139,10 @@ void marshall_withNullOrEmptyRange_shouldNotAddRangeHeader(String rangeValue) th .protocol("https") .host("example.com") .build(); - when(mockProtocolMarshaller.marshall(any(PresignedUrlGetObjectRequestWrapper.class))) + when(mockProtocolMarshaller.marshall(any(PresignedUrlDownloadRequestWrapper.class))) .thenReturn(baseRequest); - PresignedUrlGetObjectRequestWrapper request = PresignedUrlGetObjectRequestWrapper.builder() + PresignedUrlDownloadRequestWrapper request = PresignedUrlDownloadRequestWrapper.builder() .url(testUrl) .range(rangeValue) .build(); @@ -155,7 +155,7 @@ void marshall_withNullOrEmptyRange_shouldNotAddRangeHeader(String rangeValue) th void marshall_withNullRequest_shouldThrowException() { assertThatThrownBy(() -> marshaller.marshall(null)) .isInstanceOf(NullPointerException.class) - .hasMessageContaining("presignedUrlGetObjectRequestWrapper must not be null"); + .hasMessageContaining("presignedUrlDownloadRequestWrapper must not be null"); } @Test @@ -166,11 +166,11 @@ void marshall_withMalformedUrl_shouldThrowSdkClientException() throws Exception .protocol("https") .host("example.com") .build(); - when(mockProtocolMarshaller.marshall(any(PresignedUrlGetObjectRequestWrapper.class))) + when(mockProtocolMarshaller.marshall(any(PresignedUrlDownloadRequestWrapper.class))) .thenReturn(baseRequest); URL malformedUrl = new URL("https", "test-bucket.s3.us-east-1.amazonaws.com", -1, "/test key with spaces"); - PresignedUrlGetObjectRequestWrapper request = PresignedUrlGetObjectRequestWrapper.builder() + PresignedUrlDownloadRequestWrapper request = PresignedUrlDownloadRequestWrapper.builder() .url(malformedUrl) .build(); diff --git a/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/presignedurl/model/PresignedUrlGetObjectRequestWrapperTest.java b/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/presignedurl/model/PresignedUrlDownloadRequestWrapperTest.java similarity index 87% rename from services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/presignedurl/model/PresignedUrlGetObjectRequestWrapperTest.java rename to services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/presignedurl/model/PresignedUrlDownloadRequestWrapperTest.java index 8334050fa7c8..c1280667eb26 100644 --- a/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/presignedurl/model/PresignedUrlGetObjectRequestWrapperTest.java +++ b/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/presignedurl/model/PresignedUrlDownloadRequestWrapperTest.java @@ -26,17 +26,17 @@ import software.amazon.awssdk.core.protocol.MarshallLocation; -class PresignedUrlGetObjectRequestWrapperTest { +class PresignedUrlDownloadRequestWrapperTest { @Test void equalsAndHashCode_shouldFollowContract() { - EqualsVerifier.forClass(PresignedUrlGetObjectRequestWrapper.class) + EqualsVerifier.forClass(PresignedUrlDownloadRequestWrapper.class) .withRedefinedSuperclass() .verify(); } @Test void basicProperties_shouldWork() throws Exception { - PresignedUrlGetObjectRequestWrapper request = PresignedUrlGetObjectRequestWrapper.builder() + PresignedUrlDownloadRequestWrapper request = PresignedUrlDownloadRequestWrapper.builder() .url(new URL("https://example.com")) .range("bytes=0-100") .build(); @@ -47,7 +47,7 @@ void basicProperties_shouldWork() throws Exception { @Test void sdkFields_shouldReturnExpectedFields() throws Exception { - PresignedUrlGetObjectRequestWrapper request = PresignedUrlGetObjectRequestWrapper.builder() + PresignedUrlDownloadRequestWrapper request = PresignedUrlDownloadRequestWrapper.builder() .url(new URL("https://example.com")) .range("bytes=0-100") .build(); @@ -66,7 +66,7 @@ void sdkFields_shouldReturnExpectedFields() throws Exception { @Test void sdkFieldNameToField_shouldReturnExpectedMapping() throws Exception { - PresignedUrlGetObjectRequestWrapper request = PresignedUrlGetObjectRequestWrapper.builder() + PresignedUrlDownloadRequestWrapper request = PresignedUrlDownloadRequestWrapper.builder() .url(new URL("https://example.com")) .build(); @@ -80,7 +80,7 @@ void sdkFieldNameToField_shouldReturnExpectedMapping() throws Exception { @Test void rangeField_shouldMarshalCorrectly() throws Exception { - PresignedUrlGetObjectRequestWrapper request = PresignedUrlGetObjectRequestWrapper.builder() + PresignedUrlDownloadRequestWrapper request = PresignedUrlDownloadRequestWrapper.builder() .url(new URL("https://example.com")) .range("bytes=0-1023") .build(); diff --git a/services/s3/src/test/java/software/amazon/awssdk/services/s3/presignedurl/AsyncPresignedUrlManagerTest.java b/services/s3/src/test/java/software/amazon/awssdk/services/s3/presignedurl/AsyncPresignedUrlExtensionTest.java similarity index 89% rename from services/s3/src/test/java/software/amazon/awssdk/services/s3/presignedurl/AsyncPresignedUrlManagerTest.java rename to services/s3/src/test/java/software/amazon/awssdk/services/s3/presignedurl/AsyncPresignedUrlExtensionTest.java index 3441b46d142c..1b99c9bde491 100644 --- a/services/s3/src/test/java/software/amazon/awssdk/services/s3/presignedurl/AsyncPresignedUrlManagerTest.java +++ b/services/s3/src/test/java/software/amazon/awssdk/services/s3/presignedurl/AsyncPresignedUrlExtensionTest.java @@ -52,13 +52,13 @@ import software.amazon.awssdk.core.exception.SdkClientException; import software.amazon.awssdk.services.s3.S3AsyncClient; import software.amazon.awssdk.services.s3.model.GetObjectResponse; -import software.amazon.awssdk.services.s3.presignedurl.model.PresignedUrlGetObjectRequest; +import software.amazon.awssdk.services.s3.presignedurl.model.PresignedUrlDownloadRequest; @WireMockTest -public class AsyncPresignedUrlManagerTest { +public class AsyncPresignedUrlExtensionTest { private S3AsyncClient s3AsyncClient; - private AsyncPresignedUrlManager presignedUrlManager; + private AsyncPresignedUrlExtension presignedUrlExtension; @TempDir Path tempDir; @@ -68,7 +68,7 @@ void setup(WireMockRuntimeInfo wireMockRuntimeInfo) { s3AsyncClient = S3AsyncClient.builder() .endpointOverride(URI.create(wireMockRuntimeInfo.getHttpBaseUrl())) .build(); - presignedUrlManager = s3AsyncClient.presignedUrlManager(); + presignedUrlExtension = s3AsyncClient.presignedUrlExtension(); } @AfterEach @@ -81,8 +81,8 @@ void cleanup() { @Nested class BasicFunctionality { @Test - void givenS3AsyncClient_whenPresignedUrlManagerRequested_thenReturnsNonNullInstance() { - assertThat(presignedUrlManager).isNotNull(); + void givenS3AsyncClient_whenPresignedUrlExtensionRequested_thenReturnsNonNullInstance() { + assertThat(presignedUrlExtension).isNotNull(); } @Test @@ -94,10 +94,10 @@ void givenValidPresignedUrl_whenGetObjectCalled_thenReturnsExpectedContent(WireM stubSuccessResponse(presignedUrlPath, testContent, testETag); URL presignedUrl = createPresignedUrl(wireMockRuntimeInfo, presignedUrlPath); - PresignedUrlGetObjectRequest request = createRequest(presignedUrl); + PresignedUrlDownloadRequest request = createRequest(presignedUrl); CompletableFuture> result = - presignedUrlManager.getObject(request, AsyncResponseTransformer.toBytes()); + presignedUrlExtension.getObject(request, AsyncResponseTransformer.toBytes()); ResponseBytes response = result.get(); assertSuccessfulResponse(response, testContent, testETag); @@ -114,7 +114,7 @@ void givenValidPresignedUrl_whenGetObjectWithConsumerBuilderCalled_thenReturnsEx URL presignedUrl = createPresignedUrl(wireMockRuntimeInfo, presignedUrlPath); CompletableFuture> result = - presignedUrlManager.getObject( + presignedUrlExtension.getObject( request -> request.presignedUrl(presignedUrl), AsyncResponseTransformer.toBytes() ); @@ -138,10 +138,10 @@ void givenEmptyFile_whenGetObjectCalled_thenReturnsEmptyContent(WireMockRuntimeI .withBody(testContent))); URL presignedUrl = createPresignedUrl(wireMockRuntimeInfo, presignedUrlPath); - PresignedUrlGetObjectRequest request = createRequest(presignedUrl); + PresignedUrlDownloadRequest request = createRequest(presignedUrl); CompletableFuture> result = - presignedUrlManager.getObject(request, AsyncResponseTransformer.toBytes()); + presignedUrlExtension.getObject(request, AsyncResponseTransformer.toBytes()); ResponseBytes response = result.get(); assertThat(response).isNotNull(); @@ -166,13 +166,13 @@ void givenRangeRequest_whenGetObjectCalled_thenReturnsPartialContent(WireMockRun .withBody(expectedContent))); URL presignedUrl = createPresignedUrl(wireMockRuntimeInfo, presignedUrlPath); - PresignedUrlGetObjectRequest request = PresignedUrlGetObjectRequest.builder() + PresignedUrlDownloadRequest request = PresignedUrlDownloadRequest.builder() .presignedUrl(presignedUrl) .range(range) .build(); CompletableFuture> result = - presignedUrlManager.getObject(request, AsyncResponseTransformer.toBytes()); + presignedUrlExtension.getObject(request, AsyncResponseTransformer.toBytes()); ResponseBytes response = result.get(); assertThat(response).isNotNull(); @@ -191,12 +191,12 @@ void givenValidPresignedUrl_whenGetObjectToFileCalled_thenDownloadsFileWithExpec stubSuccessResponse(presignedUrlPath, testContent, testETag); URL presignedUrl = createPresignedUrl(wireMockRuntimeInfo, presignedUrlPath); - PresignedUrlGetObjectRequest request = createRequest(presignedUrl); + PresignedUrlDownloadRequest request = createRequest(presignedUrl); Path downloadFile = tempDir.resolve("download-" + UUID.randomUUID() + ".txt"); CompletableFuture result = - presignedUrlManager.getObject(request, downloadFile); + presignedUrlExtension.getObject(request, downloadFile); GetObjectResponse response = result.get(); assertThat(response).isNotNull(); @@ -218,7 +218,7 @@ void givenValidPresignedUrl_whenGetObjectToFileWithConsumerBuilderCalled_thenDow Path downloadFile = tempDir.resolve("consumer-download-" + UUID.randomUUID() + ".txt"); CompletableFuture result = - presignedUrlManager.getObject( + presignedUrlExtension.getObject( request -> request.presignedUrl(presignedUrl), downloadFile ); @@ -246,10 +246,10 @@ void givenEmptyFile_whenGetObjectToFileCalled_thenDownloadsEmptyFile(WireMockRun .withBody(testContent))); URL presignedUrl = createPresignedUrl(wireMockRuntimeInfo, presignedUrlPath); - PresignedUrlGetObjectRequest request = createRequest(presignedUrl); + PresignedUrlDownloadRequest request = createRequest(presignedUrl); Path downloadFile = tempDir.resolve("empty-file-" + UUID.randomUUID() + ".txt"); - CompletableFuture result = presignedUrlManager.getObject(request, downloadFile); + CompletableFuture result = presignedUrlExtension.getObject(request, downloadFile); GetObjectResponse response = result.get(); assertThat(response).isNotNull(); @@ -275,10 +275,10 @@ void givenLargeFile_whenGetObjectCalled_thenDownloadsCompleteContent(WireMockRun .withBody(largeContent))); URL presignedUrl = createPresignedUrl(wireMockRuntimeInfo, presignedUrlPath); - PresignedUrlGetObjectRequest request = createRequest(presignedUrl); + PresignedUrlDownloadRequest request = createRequest(presignedUrl); Path downloadFile = tempDir.resolve("large-file-" + UUID.randomUUID() + ".bin"); - CompletableFuture result = presignedUrlManager.getObject(request, downloadFile); + CompletableFuture result = presignedUrlExtension.getObject(request, downloadFile); GetObjectResponse response = result.get(); assertThat(response).isNotNull(); @@ -316,10 +316,10 @@ void givenNetworkError_whenGetObjectCalled_thenRetriesAndEventuallySucceeds(Wire .withBody(testContent))); URL presignedUrl = createPresignedUrl(wireMockRuntimeInfo, presignedUrlPath); - PresignedUrlGetObjectRequest request = createRequest(presignedUrl); + PresignedUrlDownloadRequest request = createRequest(presignedUrl); CompletableFuture> result = - presignedUrlManager.getObject(request, AsyncResponseTransformer.toBytes()); + presignedUrlExtension.getObject(request, AsyncResponseTransformer.toBytes()); ResponseBytes response = result.get(); assertSuccessfulResponse(response, testContent, testETag); @@ -351,10 +351,10 @@ void givenTemporaryFailure_whenGetObjectCalled_thenRetriesAndSucceeds(WireMockRu .withBody(testContent))); URL presignedUrl = createPresignedUrl(wireMockRuntimeInfo, presignedUrlPath); - PresignedUrlGetObjectRequest request = createRequest(presignedUrl); + PresignedUrlDownloadRequest request = createRequest(presignedUrl); CompletableFuture> result = - presignedUrlManager.getObject(request, AsyncResponseTransformer.toBytes()); + presignedUrlExtension.getObject(request, AsyncResponseTransformer.toBytes()); ResponseBytes response = result.get(); assertSuccessfulResponse(response, testContent, testETag); @@ -373,10 +373,10 @@ void givenMalformedResponse_whenGetObjectCalled_thenCompletableFutureCompletesEx .withFault(Fault.CONNECTION_RESET_BY_PEER))); URL presignedUrl = createPresignedUrl(wireMockRuntimeInfo, presignedUrlPath); - PresignedUrlGetObjectRequest request = createRequest(presignedUrl); + PresignedUrlDownloadRequest request = createRequest(presignedUrl); CompletableFuture> result = - presignedUrlManager.getObject(request, AsyncResponseTransformer.toBytes()); + presignedUrlExtension.getObject(request, AsyncResponseTransformer.toBytes()); try { result.get(); throw new AssertionError("Expected exception was not thrown"); @@ -401,12 +401,12 @@ void givenNotFoundError_whenGetObjectCalled_thenCompletableFutureCompletesExcept throw new RuntimeException(e); } - PresignedUrlGetObjectRequest request = PresignedUrlGetObjectRequest.builder() + PresignedUrlDownloadRequest request = PresignedUrlDownloadRequest.builder() .presignedUrl(presignedUrl) .build(); CompletableFuture> result = - presignedUrlManager.getObject(request, AsyncResponseTransformer.toBytes()); + presignedUrlExtension.getObject(request, AsyncResponseTransformer.toBytes()); assertThatThrownBy(() -> result.get()) .isInstanceOf(ExecutionException.class); @@ -428,12 +428,12 @@ void givenAccessDeniedError_whenGetObjectCalled_thenCompletableFutureCompletesEx throw new RuntimeException(e); } - PresignedUrlGetObjectRequest request = PresignedUrlGetObjectRequest.builder() + PresignedUrlDownloadRequest request = PresignedUrlDownloadRequest.builder() .presignedUrl(presignedUrl) .build(); CompletableFuture> result = - presignedUrlManager.getObject(request, AsyncResponseTransformer.toBytes()); + presignedUrlExtension.getObject(request, AsyncResponseTransformer.toBytes()); assertThatThrownBy(() -> result.get()) .isInstanceOf(ExecutionException.class); @@ -464,9 +464,9 @@ void givenMultipleRequests_whenGetObjectCalledConcurrently_thenAllDownloadsCompl List>> futures = new ArrayList<>(); for (int i = 0; i < concurrentRequests; i++) { URL presignedUrl = createPresignedUrl(wireMockRuntimeInfo, paths.get(i)); - PresignedUrlGetObjectRequest request = createRequest(presignedUrl); + PresignedUrlDownloadRequest request = createRequest(presignedUrl); - futures.add(presignedUrlManager.getObject(request, AsyncResponseTransformer.toBytes())); + futures.add(presignedUrlExtension.getObject(request, AsyncResponseTransformer.toBytes())); } CompletableFuture allFutures = CompletableFuture.allOf(futures.toArray(new CompletableFuture[0])); @@ -488,8 +488,8 @@ private URL createPresignedUrl(WireMockRuntimeInfo wireMockRuntimeInfo, String p } } - private PresignedUrlGetObjectRequest createRequest(URL presignedUrl) { - return PresignedUrlGetObjectRequest.builder() + private PresignedUrlDownloadRequest createRequest(URL presignedUrl) { + return PresignedUrlDownloadRequest.builder() .presignedUrl(presignedUrl) .build(); } diff --git a/services/s3/src/test/java/software/amazon/awssdk/services/s3/presignedurl/model/PresignedUrlGetObjectRequestTest.java b/services/s3/src/test/java/software/amazon/awssdk/services/s3/presignedurl/model/PresignedUrlDownloadRequestTest.java similarity index 83% rename from services/s3/src/test/java/software/amazon/awssdk/services/s3/presignedurl/model/PresignedUrlGetObjectRequestTest.java rename to services/s3/src/test/java/software/amazon/awssdk/services/s3/presignedurl/model/PresignedUrlDownloadRequestTest.java index 14048f2fe1a7..1cc7b34364f0 100644 --- a/services/s3/src/test/java/software/amazon/awssdk/services/s3/presignedurl/model/PresignedUrlGetObjectRequestTest.java +++ b/services/s3/src/test/java/software/amazon/awssdk/services/s3/presignedurl/model/PresignedUrlDownloadRequestTest.java @@ -21,18 +21,18 @@ import nl.jqno.equalsverifier.EqualsVerifier; import org.junit.jupiter.api.Test; -class PresignedUrlGetObjectRequestTest { +class PresignedUrlDownloadRequestTest { @Test void equalsAndHashCode_shouldFollowContract() { - EqualsVerifier.forClass(PresignedUrlGetObjectRequest.class) + EqualsVerifier.forClass(PresignedUrlDownloadRequest.class) .verify(); } @Test void builder_shouldCreateRequestWithAllFields() throws Exception { URL url = new URL("https://example.com"); - PresignedUrlGetObjectRequest request = PresignedUrlGetObjectRequest.builder() + PresignedUrlDownloadRequest request = PresignedUrlDownloadRequest.builder() .presignedUrl(url) .range("bytes=0-100") .build(); @@ -44,7 +44,7 @@ void builder_shouldCreateRequestWithAllFields() throws Exception { @Test void builder_shouldCreateRequestWithOnlyRequiredFields() throws Exception { URL url = new URL("https://example.com"); - PresignedUrlGetObjectRequest request = PresignedUrlGetObjectRequest.builder() + PresignedUrlDownloadRequest request = PresignedUrlDownloadRequest.builder() .presignedUrl(url) .build(); @@ -55,12 +55,12 @@ void builder_shouldCreateRequestWithOnlyRequiredFields() throws Exception { @Test void toBuilder_shouldCreateBuilderFromExistingRequest() throws Exception { URL url = new URL("https://example.com"); - PresignedUrlGetObjectRequest original = PresignedUrlGetObjectRequest.builder() + PresignedUrlDownloadRequest original = PresignedUrlDownloadRequest.builder() .presignedUrl(url) .range("bytes=0-100") .build(); - PresignedUrlGetObjectRequest copy = original.toBuilder().build(); + PresignedUrlDownloadRequest copy = original.toBuilder().build(); assertThat(copy.presignedUrl()).isEqualTo(original.presignedUrl()); assertThat(copy.range()).isEqualTo(original.range()); @@ -70,12 +70,12 @@ void toBuilder_shouldCreateBuilderFromExistingRequest() throws Exception { void toBuilder_shouldAllowModification() throws Exception { URL url1 = new URL("https://example.com"); URL url2 = new URL("https://other.com"); - PresignedUrlGetObjectRequest original = PresignedUrlGetObjectRequest.builder() + PresignedUrlDownloadRequest original = PresignedUrlDownloadRequest.builder() .presignedUrl(url1) .range("bytes=0-100") .build(); - PresignedUrlGetObjectRequest modified = original.toBuilder() + PresignedUrlDownloadRequest modified = original.toBuilder() .presignedUrl(url2) .range("bytes=200-300") .build(); @@ -92,7 +92,7 @@ void toString_shouldContainActualFieldValues() throws Exception { URL url = new URL("https://example.com"); String range = "bytes=0-100"; - PresignedUrlGetObjectRequest request = PresignedUrlGetObjectRequest.builder() + PresignedUrlDownloadRequest request = PresignedUrlDownloadRequest.builder() .presignedUrl(url) .range(range) .build(); @@ -109,7 +109,7 @@ void toString_shouldContainActualFieldValues() throws Exception { @Test void serializableBuilderClass_shouldReturnCorrectClass() { - assertThat(PresignedUrlGetObjectRequest.serializableBuilderClass()) - .isEqualTo(PresignedUrlGetObjectRequest.BuilderImpl.class); + assertThat(PresignedUrlDownloadRequest.serializableBuilderClass()) + .isEqualTo(PresignedUrlDownloadRequest.BuilderImpl.class); } } From 365eaefcb98c740367b0d6d79aeb5eb4aeda4aca Mon Sep 17 00:00:00 2001 From: jencymaryjoseph <35571282+jencymaryjoseph@users.noreply.github.com> Date: Tue, 29 Jul 2025 08:28:25 -0700 Subject: [PATCH 09/30] Add AsyncPresignedUrlManager integration tests (#6290) * Amazon DataZone Update: This release adds support for 1) highlighting relevant text in returned results for Search and SearchListings APIs and 2) returning aggregated counts of values for specified attributes for SearchListings API. * Amazon Omics Update: Add Git integration and README support for HealthOmics workflows * Updated endpoints.json and partitions.json. * Release 2.32.8. Updated CHANGELOG.md, README.md and all pom.xml. * Update to next snapshot version: 2.32.9-SNAPSHOT * Amazon AppIntegrations Service Update: Amazon AppIntegrations introduces new configuration capabilities to enable customers to manage iframe permissions, control application refresh behavior (per contact or per browser/cross-contact), and run background applications (service). * AWS Key Management Service Update: Doc only update: fixed grammatical errors. * AWS Config Update: Documentation improvements have been made to the EvaluationModel and DescribeConfigurationRecorders APIs. * AWS End User Messaging Social Update: This release introduces new WhatsApp template management APIs that enable customers to programmatically create and submit templates for approval, monitor approval status, and manage the complete template lifecycle * AWS Budgets Update: Adds IPv6 and PrivateLink support for AWS Budgets in IAD. * Amazon Simple Queue Service Update: Documentation updates for Amazon SQS fair queues feature. * AWS Elemental MediaPackage v2 Update: This release adds support for specifying a preferred input for channels using CMAF ingest. * Amazon Elastic Compute Cloud Update: Transit Gateway native integration with AWS Network Firewall. Adding new enum value for the new Transit Gateway Attachment type. * Release 2.32.9. Updated CHANGELOG.md, README.md and all pom.xml. * Update to next snapshot version: 2.32.10-SNAPSHOT * AWS IoT SiteWise Update: Add support for native anomaly detection in IoT SiteWise using new Computation Model APIs * Amazon OpenSearch Ingestion Update: Add Pipeline Role Arn as an optional parameter to the create / update pipeline APIs as an alternative to passing in the pipeline configuration body * AWS Direct Connect Update: Enable MACSec support and features on Interconnects. * Updated endpoints.json and partitions.json. * Release 2.32.10. Updated CHANGELOG.md, README.md and all pom.xml. * Update to next snapshot version: 2.32.11-SNAPSHOT * Rebase #6269: Update PresignedUrlManager to PresignedUrlExtension naming * Integ tests * Refactor AsyncPresignedUrlManager integration tests to extend S3IntegrationTestBase * updated test method names to follow testing guidelines * simplified tests * added checksum test for largeobjects * rename in test files, address nit comment, fix rebase issues --------- Co-authored-by: AWS <> --- .../poet/client/test-presignedurl-async.java | 4 +- ...cPresignedUrlExtensionIntegrationTest.java | 32 ++ .../AsyncPresignedUrlExtensionTestSuite.java | 303 ++++++++++++++++++ 3 files changed, 337 insertions(+), 2 deletions(-) create mode 100644 services/s3/src/it/java/software/amazon/awssdk/services/s3/presignedurl/AsyncPresignedUrlExtensionIntegrationTest.java create mode 100644 services/s3/src/it/java/software/amazon/awssdk/services/s3/presignedurl/AsyncPresignedUrlExtensionTestSuite.java diff --git a/codegen/src/test/resources/software/amazon/awssdk/codegen/poet/client/test-presignedurl-async.java b/codegen/src/test/resources/software/amazon/awssdk/codegen/poet/client/test-presignedurl-async.java index c5fcbdb05db3..dcef20b7814b 100644 --- a/codegen/src/test/resources/software/amazon/awssdk/codegen/poet/client/test-presignedurl-async.java +++ b/codegen/src/test/resources/software/amazon/awssdk/codegen/poet/client/test-presignedurl-async.java @@ -206,10 +206,10 @@ private void updateRetryStrategyClientConfiguration(SdkClientConfiguration.Build private SdkClientConfiguration updateSdkClientConfiguration(SdkRequest request, SdkClientConfiguration clientConfiguration) { List plugins = request.overrideConfiguration().map(c -> c.plugins()).orElse(Collections.emptyList()); - SdkClientConfiguration.Builder configuration = clientConfiguration.toBuilder(); if (plugins.isEmpty()) { - return configuration.build(); + return clientConfiguration; } + SdkClientConfiguration.Builder configuration = clientConfiguration.toBuilder(); JsonServiceClientConfigurationBuilder serviceConfigBuilder = new JsonServiceClientConfigurationBuilder(configuration); for (SdkPlugin plugin : plugins) { plugin.configureClient(serviceConfigBuilder); diff --git a/services/s3/src/it/java/software/amazon/awssdk/services/s3/presignedurl/AsyncPresignedUrlExtensionIntegrationTest.java b/services/s3/src/it/java/software/amazon/awssdk/services/s3/presignedurl/AsyncPresignedUrlExtensionIntegrationTest.java new file mode 100644 index 000000000000..1ec2cf2df702 --- /dev/null +++ b/services/s3/src/it/java/software/amazon/awssdk/services/s3/presignedurl/AsyncPresignedUrlExtensionIntegrationTest.java @@ -0,0 +1,32 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ +package software.amazon.awssdk.services.s3.presignedurl; + +import org.junit.jupiter.api.BeforeAll; +import software.amazon.awssdk.services.s3.S3AsyncClient; + +public class AsyncPresignedUrlExtensionIntegrationTest extends AsyncPresignedUrlExtensionTestSuite { + + @BeforeAll + static void setUpIntegrationTest() { + S3AsyncClient s3AsyncClient = s3AsyncClientBuilder().build(); + presignedUrlExtension = s3AsyncClient.presignedUrlExtension(); + } + + @Override + protected S3AsyncClient createS3AsyncClient() { + return s3AsyncClientBuilder().build(); + } +} diff --git a/services/s3/src/it/java/software/amazon/awssdk/services/s3/presignedurl/AsyncPresignedUrlExtensionTestSuite.java b/services/s3/src/it/java/software/amazon/awssdk/services/s3/presignedurl/AsyncPresignedUrlExtensionTestSuite.java new file mode 100644 index 000000000000..eee244ca3485 --- /dev/null +++ b/services/s3/src/it/java/software/amazon/awssdk/services/s3/presignedurl/AsyncPresignedUrlExtensionTestSuite.java @@ -0,0 +1,303 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ +package software.amazon.awssdk.services.s3.presignedurl; + +import static org.apache.commons.lang3.RandomStringUtils.randomAscii; +import static org.assertj.core.api.Assertions.assertThat; + +import java.io.ByteArrayInputStream; +import java.net.URL; +import java.nio.charset.StandardCharsets; +import java.nio.file.Path; +import java.time.Duration; +import java.util.ArrayList; +import java.util.List; +import java.util.UUID; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.TimeUnit; +import java.util.stream.Stream; + +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +import software.amazon.awssdk.core.ResponseBytes; +import software.amazon.awssdk.core.async.AsyncRequestBody; +import software.amazon.awssdk.core.async.AsyncResponseTransformer; +import software.amazon.awssdk.metrics.MetricCollection; +import software.amazon.awssdk.metrics.MetricPublisher; +import software.amazon.awssdk.services.s3.S3AsyncClient; +import software.amazon.awssdk.services.s3.S3IntegrationTestBase; +import software.amazon.awssdk.services.s3.model.DeleteObjectRequest; +import software.amazon.awssdk.services.s3.model.GetObjectResponse; +import software.amazon.awssdk.services.s3.model.PutObjectRequest; +import software.amazon.awssdk.services.s3.presigner.S3Presigner; +import software.amazon.awssdk.services.s3.presigner.model.PresignedGetObjectRequest; +import software.amazon.awssdk.services.s3.presignedurl.model.PresignedUrlDownloadRequest; +import software.amazon.awssdk.services.s3.utils.S3TestUtils; +import software.amazon.awssdk.testutils.service.S3BucketUtils; +import software.amazon.awssdk.utils.Md5Utils; + +/** + * Abstract test suite for AsyncPresignedUrlExtension integration tests. + */ +public abstract class AsyncPresignedUrlExtensionTestSuite extends S3IntegrationTestBase { + protected static S3Presigner presigner; + protected static AsyncPresignedUrlExtension presignedUrlExtension; + protected static String testBucket; + + @TempDir + static Path temporaryFolder; + + protected static String testGetObjectKey; + protected static String testLargeObjectKey; + protected static String testObjectContent; + protected static byte[] testLargeObjectContent; + protected static String expectedLargeObjectMd5; + + protected abstract S3AsyncClient createS3AsyncClient(); + + @BeforeAll + static void setUpTestSuite() throws Exception { + setUp(); + + presigner = S3Presigner.builder() + .region(DEFAULT_REGION) + .credentialsProvider(CREDENTIALS_PROVIDER_CHAIN) + .build(); + testBucket = S3BucketUtils.temporaryBucketName("async-presigned-url-extension-test"); + createBucket(testBucket); + testGetObjectKey = generateRandomObjectKey(); + testLargeObjectKey = generateRandomObjectKey() + "-large"; + testObjectContent = "Hello AsyncPresignedUrlExtension Integration Test"; + testLargeObjectContent = randomAscii(5 * 1024 * 1024).getBytes(StandardCharsets.UTF_8); + + try (ByteArrayInputStream originalStream = new ByteArrayInputStream(testLargeObjectContent)) { + expectedLargeObjectMd5 = Md5Utils.md5AsBase64(originalStream); + } catch (Exception e) { + throw new RuntimeException("Failed to compute MD5 for test data", e); + } + + S3TestUtils.putObject(AsyncPresignedUrlExtensionTestSuite.class, s3, testBucket, testGetObjectKey, testObjectContent); + s3Async.putObject( + PutObjectRequest.builder() + .bucket(testBucket) + .key(testLargeObjectKey) + .build(), + AsyncRequestBody.fromBytes(testLargeObjectContent) + ).join(); + S3TestUtils.addCleanupTask(AsyncPresignedUrlExtensionTestSuite.class, () -> { + s3.deleteObject(DeleteObjectRequest.builder() + .bucket(testBucket) + .key(testGetObjectKey) + .build()); + s3.deleteObject(DeleteObjectRequest.builder() + .bucket(testBucket) + .key(testLargeObjectKey) + .build()); + deleteBucketAndAllContents(testBucket); + }); + } + + @AfterAll + static void tearDownTestSuite() { + try { + S3TestUtils.runCleanupTasks(AsyncPresignedUrlExtensionTestSuite.class); + } catch (Exception e) { + } + if (presigner != null) { + presigner.close(); + } + cleanUpResources(); + } + + @ParameterizedTest(name = "{0}") + @MethodSource("basicFunctionalityTestData") + void getObject_withValidPresignedUrl_returnsContent(String testDescription, + String objectKey, + String expectedContent) throws Exception { + PresignedUrlDownloadRequest request = createRequestForKey(objectKey); + + CompletableFuture> future = + presignedUrlExtension.getObject(request, AsyncResponseTransformer.toBytes()); + ResponseBytes response = future.get(); + + assertThat(response).isNotNull(); + if (expectedContent != null) { + assertThat(response.asUtf8String()).isEqualTo(expectedContent); + assertThat(response.response().contentLength()).isEqualTo(expectedContent.length()); + } else { + try (ByteArrayInputStream downloadedStream = new ByteArrayInputStream(response.asByteArray())) { + String downloadedMd5 = Md5Utils.md5AsBase64(downloadedStream); + assertThat(downloadedMd5).isEqualTo(expectedLargeObjectMd5); + assertThat(response.asByteArray().length).isEqualTo(testLargeObjectContent.length); + } + assertThat(response.response()).isNotNull(); + } + } + + @Test + void getObject_withValidPresignedUrl_savesContentToFile() throws Exception { + PresignedUrlDownloadRequest request = createRequestForKey(testGetObjectKey); + Path downloadFile = temporaryFolder.resolve("download-" + UUID.randomUUID() + ".txt"); + CompletableFuture future = + presignedUrlExtension.getObject(request, downloadFile); + GetObjectResponse response = future.get(); + + assertThat(response).isNotNull(); + assertThat(downloadFile).exists(); + assertThat(downloadFile).hasContent(testObjectContent); + } + + @ParameterizedTest(name = "{0}") + @MethodSource("rangeTestData") + void getObject_withRangeRequest_returnsSpecifiedRange(String testDescription, + String range, + String expectedContent) throws Exception { + PresignedUrlDownloadRequest request = createRequestForKey(testGetObjectKey, range); + + CompletableFuture> future = + presignedUrlExtension.getObject(request, AsyncResponseTransformer.toBytes()); + ResponseBytes response = future.get(); + + assertThat(response.asUtf8String()).isEqualTo(expectedContent); + } + + @Test + void getObject_withMultipleRangeRequestsConcurrently_returnsCorrectContent() throws Exception { + String concurrentTestKey = uploadTestObject("concurrent-test", "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ"); + List>> futures = new ArrayList<>(); + + futures.add(presignedUrlExtension.getObject( + createRequestForKey(concurrentTestKey, "bytes=0-8"), // "012345678" + AsyncResponseTransformer.toBytes())); + futures.add(presignedUrlExtension.getObject( + createRequestForKey(concurrentTestKey, "bytes=9-17"), // "9ABCDEFGH" + AsyncResponseTransformer.toBytes())); + futures.add(presignedUrlExtension.getObject( + createRequestForKey(concurrentTestKey, "bytes=18-26"), // "IJKLMNOPQ" + AsyncResponseTransformer.toBytes())); + futures.add(presignedUrlExtension.getObject( + createRequestForKey(concurrentTestKey, "bytes=27-35"), // "RSTUVWXYZ" + AsyncResponseTransformer.toBytes())); + + CompletableFuture allFutures = CompletableFuture.allOf( + futures.toArray(new CompletableFuture[0])); + allFutures.get(30, TimeUnit.SECONDS); + + StringBuilder result = new StringBuilder(); + for (CompletableFuture> future : futures) { + result.append(future.get().asUtf8String()); + } + + assertThat(result.toString()).isEqualTo("0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ"); + } + + @Test + void getObject_withLargeObjectToFile_savesCompleteContentAndCollectsMetrics() throws Exception { + List collectedMetrics = new ArrayList<>(); + MetricPublisher metricPublisher = new MetricPublisher() { + @Override + public void publish(MetricCollection metricCollection) { + collectedMetrics.add(metricCollection); + } + @Override + public void close() {} + }; + + try (S3AsyncClient clientWithMetrics = s3AsyncClientBuilder() + .overrideConfiguration(o -> o.addMetricPublisher(metricPublisher)) + .build()) { + + AsyncPresignedUrlExtension metricsExtension = clientWithMetrics.presignedUrlExtension(); + PresignedUrlDownloadRequest request = createRequestForKey(testLargeObjectKey); + Path downloadFile = temporaryFolder.resolve("large-download-with-metrics-" + UUID.randomUUID() + ".bin"); + + CompletableFuture future = + metricsExtension.getObject(request, downloadFile); + GetObjectResponse response = future.get(60, TimeUnit.SECONDS); + + assertThat(response).isNotNull(); + assertThat(downloadFile).exists(); + assertThat(downloadFile.toFile().length()).isEqualTo(testLargeObjectContent.length); + assertThat(collectedMetrics).isNotEmpty(); + } + } + + static Stream basicFunctionalityTestData() { + return Stream.of( + Arguments.of("getObject_withValidUrl_returnsContent", + testGetObjectKey, testObjectContent), + Arguments.of("getObject_withValidLargeObjectUrl_returnsContent", + testLargeObjectKey, null) + ); + } + + static Stream rangeTestData() { + String content = "Hello AsyncPresignedUrlExtension Integration Test"; + return Stream.of( + Arguments.of("getObject_withPrefix10BytesRange_returnsFirst10Bytes", + "bytes=0-9", content.substring(0, 10)), + Arguments.of("getObject_withSuffix10BytesRange_returnsLast10Bytes", + "bytes=-10", content.substring(content.length() - 10)), + Arguments.of("getObject_withMiddle10BytesRange_returnsMiddle10Bytes", + "bytes=10-19", content.substring(10, 20)), + Arguments.of("getObject_withSingleByteRange_returnsSingleByte", + "bytes=0-0", content.substring(0, 1)) + ); + } + + // Helper methods + private static String generateRandomObjectKey() { + return "async-presigned-url-extension-test-" + UUID.randomUUID(); + } + + private PresignedUrlDownloadRequest createRequestForKey(String key) { + return PresignedUrlDownloadRequest.builder() + .presignedUrl(createPresignedUrl(key)) + .build(); + } + + private PresignedUrlDownloadRequest createRequestForKey(String key, String range) { + return PresignedUrlDownloadRequest.builder() + .presignedUrl(createPresignedUrl(key)) + .range(range) + .build(); + } + + private URL createPresignedUrl(String key) { + PresignedGetObjectRequest presignedRequest = presigner.presignGetObject(r -> r + .getObjectRequest(req -> req.bucket(testBucket).key(key)) + .signatureDuration(Duration.ofMinutes(10))); + return presignedRequest.url(); + } + + private String uploadTestObject(String keyPrefix, String content) { + String key = keyPrefix + "-" + UUID.randomUUID(); + S3TestUtils.putObject(AsyncPresignedUrlExtensionTestSuite.class, s3, testBucket, key, content); + + S3TestUtils.addCleanupTask(AsyncPresignedUrlExtensionTestSuite.class, () -> { + s3.deleteObject(DeleteObjectRequest.builder() + .bucket(testBucket) + .key(key) + .build()); + }); + return key; + } +} From 96a06bfcbe8f4d57605edd6ba14d32efed57897e Mon Sep 17 00:00:00 2001 From: jencymaryjoseph <35571282+jencymaryjoseph@users.noreply.github.com> Date: Tue, 12 Aug 2025 13:07:30 -0700 Subject: [PATCH 10/30] Add multipart presigned URL download support to MultipartS3AsyncClient (#6318) * Add multipart presigned URL download support with range-based routing * Renamed AsyncPresignedUrlExtension variable * Commented the failing test with TODO --- .../MultipartAsyncPresignedUrlExtension.java | 57 ++++++++++++++ .../multipart/MultipartS3AsyncClient.java | 12 ++- .../multipart/PresignedUrlDownloadHelper.java | 77 +++++++++++++++++++ .../multipart/MultipartS3AsyncClientTest.java | 54 +++++++++++++ 4 files changed, 198 insertions(+), 2 deletions(-) create mode 100644 services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/MultipartAsyncPresignedUrlExtension.java create mode 100644 services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/PresignedUrlDownloadHelper.java diff --git a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/MultipartAsyncPresignedUrlExtension.java b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/MultipartAsyncPresignedUrlExtension.java new file mode 100644 index 000000000000..3e4b9ade0033 --- /dev/null +++ b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/MultipartAsyncPresignedUrlExtension.java @@ -0,0 +1,57 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.services.s3.internal.multipart; + +import java.util.concurrent.CompletableFuture; +import software.amazon.awssdk.annotations.SdkInternalApi; +import software.amazon.awssdk.core.async.AsyncResponseTransformer; +import software.amazon.awssdk.services.s3.S3AsyncClient; +import software.amazon.awssdk.services.s3.model.GetObjectResponse; +import software.amazon.awssdk.services.s3.presignedurl.AsyncPresignedUrlExtension; +import software.amazon.awssdk.services.s3.presignedurl.model.PresignedUrlDownloadRequest; +import software.amazon.awssdk.utils.Validate; + +/** + * An {@link AsyncPresignedUrlExtension} that automatically converts presigned URL downloads + * to multipart downloads. + */ +@SdkInternalApi +public class MultipartAsyncPresignedUrlExtension implements AsyncPresignedUrlExtension { + private final PresignedUrlDownloadHelper downloadHelper; + + public MultipartAsyncPresignedUrlExtension( + S3AsyncClient s3AsyncClient, + AsyncPresignedUrlExtension asyncPresignedUrlExtension, + long bufferSizeInBytes, + long partSizeInBytes) { + Validate.paramNotNull(s3AsyncClient, "s3AsyncClient"); + Validate.paramNotNull(asyncPresignedUrlExtension, "asyncPresignedUrlExtension"); + this.downloadHelper = new PresignedUrlDownloadHelper( + s3AsyncClient, + asyncPresignedUrlExtension, + bufferSizeInBytes, + partSizeInBytes); + } + + @Override + public CompletableFuture getObject( + PresignedUrlDownloadRequest presignedUrlDownloadRequest, + AsyncResponseTransformer asyncResponseTransformer) { + Validate.paramNotNull(presignedUrlDownloadRequest, "presignedUrlDownloadRequest"); + Validate.paramNotNull(asyncResponseTransformer, "asyncResponseTransformer"); + return downloadHelper.downloadObject(presignedUrlDownloadRequest, asyncResponseTransformer); + } +} \ No newline at end of file diff --git a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/MultipartS3AsyncClient.java b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/MultipartS3AsyncClient.java index bcb370e8286c..6d3aca4970e1 100644 --- a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/MultipartS3AsyncClient.java +++ b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/MultipartS3AsyncClient.java @@ -53,6 +53,8 @@ public final class MultipartS3AsyncClient extends DelegatingS3AsyncClient { private final CopyObjectHelper copyObjectHelper; private final DownloadObjectHelper downloadObjectHelper; private final boolean checksumEnabled; + private final long apiCallBufferSize; + private final long minPartSizeInBytes; private MultipartS3AsyncClient(S3AsyncClient delegate, MultipartConfiguration multipartConfiguration, boolean checksumEnabled) { @@ -64,6 +66,8 @@ private MultipartS3AsyncClient(S3AsyncClient delegate, MultipartConfiguration mu long threshold = resolver.thresholdInBytes(); long apiCallBufferSize = resolver.apiCallBufferSize(); int maxInFlightParts = resolver.maxInFlightParts(); + this.apiCallBufferSize = apiCallBufferSize; + this.minPartSizeInBytes = minPartSizeInBytes; mpuHelper = new UploadObjectHelper(delegate, resolver); copyObjectHelper = new CopyObjectHelper(delegate, minPartSizeInBytes, threshold); downloadObjectHelper = new DownloadObjectHelper(delegate, apiCallBufferSize, maxInFlightParts); @@ -115,7 +119,11 @@ protected CompletableFuture invokeOperat @Override public AsyncPresignedUrlExtension presignedUrlExtension() { - // TODO: Implement presigned URL extension support for multipart client - throw new UnsupportedOperationException("Presigned URL extension is not supported for multipart client"); + AsyncPresignedUrlExtension delegateExtension = ((S3AsyncClient) delegate()).presignedUrlExtension(); + return new MultipartAsyncPresignedUrlExtension( + (S3AsyncClient) delegate(), + delegateExtension, + apiCallBufferSize, + minPartSizeInBytes); } } diff --git a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/PresignedUrlDownloadHelper.java b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/PresignedUrlDownloadHelper.java new file mode 100644 index 000000000000..cfb94e8a0242 --- /dev/null +++ b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/PresignedUrlDownloadHelper.java @@ -0,0 +1,77 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.services.s3.internal.multipart; + +import java.util.concurrent.CompletableFuture; +import software.amazon.awssdk.annotations.SdkInternalApi; +import software.amazon.awssdk.core.SplittingTransformerConfiguration; +import software.amazon.awssdk.core.async.AsyncResponseTransformer; +import software.amazon.awssdk.services.s3.S3AsyncClient; +import software.amazon.awssdk.services.s3.model.GetObjectResponse; +import software.amazon.awssdk.services.s3.presignedurl.AsyncPresignedUrlExtension; +import software.amazon.awssdk.services.s3.presignedurl.model.PresignedUrlDownloadRequest; +import software.amazon.awssdk.utils.Logger; +import software.amazon.awssdk.utils.Validate; + +@SdkInternalApi +public class PresignedUrlDownloadHelper { + private static final Logger log = Logger.loggerFor(PresignedUrlDownloadHelper.class); + + private final S3AsyncClient s3AsyncClient; + private final AsyncPresignedUrlExtension asyncPresignedUrlExtension; + private final long bufferSizeInBytes; + private final long configuredPartSizeInBytes; + + public PresignedUrlDownloadHelper(S3AsyncClient s3AsyncClient, + AsyncPresignedUrlExtension asyncPresignedUrlExtension, + long bufferSizeInBytes, + long configuredPartSizeInBytes) { + this.s3AsyncClient = Validate.paramNotNull(s3AsyncClient, "s3AsyncClient"); + this.asyncPresignedUrlExtension = Validate.paramNotNull(asyncPresignedUrlExtension, "asyncPresignedUrlExtension"); + this.bufferSizeInBytes = Validate.isPositive(bufferSizeInBytes, "bufferSizeInBytes"); + this.configuredPartSizeInBytes = Validate.isPositive(configuredPartSizeInBytes, "configuredPartSizeInBytes"); + } + + public CompletableFuture downloadObject( + PresignedUrlDownloadRequest presignedRequest, + AsyncResponseTransformer asyncResponseTransformer) { + + Validate.paramNotNull(presignedRequest, "presignedRequest"); + Validate.paramNotNull(asyncResponseTransformer, "asyncResponseTransformer"); + + if (presignedRequest.range() != null) { + log.debug(() -> "Using single part download because presigned URL request range is included in the request. range = " + + presignedRequest.range()); + return asyncPresignedUrlExtension.getObject(presignedRequest, asyncResponseTransformer); + } + + SplittingTransformerConfiguration splittingConfig = SplittingTransformerConfiguration.builder() + .bufferSizeInBytes(bufferSizeInBytes) + .build(); + AsyncResponseTransformer.SplitResult split = + asyncResponseTransformer.split(splittingConfig); + // TODO: PresignedUrlMultipartDownloaderSubscriber needs to be implemented in next PR + // PresignedUrlMultipartDownloaderSubscriber subscriber = + // new PresignedUrlMultipartDownloaderSubscriber( + // s3AsyncClient, + // presignedRequest, + // configuredPartSizeInBytes); + // + // split.publisher().subscribe(subscriber); + // return split.resultFuture(); + throw new UnsupportedOperationException("Multipart presigned URL download not yet implemented - TODO in next PR"); + } +} diff --git a/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/multipart/MultipartS3AsyncClientTest.java b/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/multipart/MultipartS3AsyncClientTest.java index 848d7259124b..d92fd69ac3c9 100644 --- a/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/multipart/MultipartS3AsyncClientTest.java +++ b/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/multipart/MultipartS3AsyncClientTest.java @@ -15,13 +15,17 @@ package software.amazon.awssdk.services.s3.internal.multipart; +import static org.assertj.core.api.AssertionsForClassTypes.assertThat; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; +import java.net.MalformedURLException; +import java.net.URL; import org.junit.jupiter.api.Test; import software.amazon.awssdk.core.ResponseBytes; import software.amazon.awssdk.core.SplittingTransformerConfiguration; @@ -30,6 +34,8 @@ import software.amazon.awssdk.services.s3.model.GetObjectRequest; import software.amazon.awssdk.services.s3.model.GetObjectResponse; import software.amazon.awssdk.services.s3.multipart.MultipartConfiguration; +import software.amazon.awssdk.services.s3.presignedurl.AsyncPresignedUrlExtension; +import software.amazon.awssdk.services.s3.presignedurl.model.PresignedUrlDownloadRequest; class MultipartS3AsyncClientTest { @@ -64,4 +70,52 @@ void partManuallySpecified_shouldBypassMultipart() { verify(mockTransformer, never()).split(any(SplittingTransformerConfiguration.class)); verify(mockDelegate, times(1)).getObject(any(GetObjectRequest.class), eq(mockTransformer)); } + + @Test + void presignedUrlExtension_rangeSpecified_shouldBypassMultipart() throws MalformedURLException { + S3AsyncClient mockDelegate = mock(S3AsyncClient.class); + AsyncPresignedUrlExtension mockDelegateExtension = mock(AsyncPresignedUrlExtension.class); + AsyncResponseTransformer mockTransformer = mock(AsyncResponseTransformer.class); + PresignedUrlDownloadRequest req = PresignedUrlDownloadRequest.builder() + .presignedUrl(new URL("https://s3.amazonaws.com/bucket/key?signature=abc")) + .range("bytes=0-1023") + .build(); + when(mockDelegate.presignedUrlExtension()).thenReturn(mockDelegateExtension); + S3AsyncClient s3AsyncClient = MultipartS3AsyncClient.create(mockDelegate, MultipartConfiguration.builder().build(), true); + s3AsyncClient.presignedUrlExtension().getObject(req, mockTransformer); + verify(mockTransformer, never()).split(any(SplittingTransformerConfiguration.class)); + verify(mockDelegateExtension, times(1)).getObject(eq(req), eq(mockTransformer)); + } + + // TODO: Enable this test once PresignedUrlMultipartDownloaderSubscriber is implemented + // // Currently fails because multipart presigned URL download throws UnsupportedOperationException + // @Test + // void presignedUrlExtension_noRange_shouldUseMultipart() throws MalformedURLException { + // S3AsyncClient mockDelegate = mock(S3AsyncClient.class); + // AsyncPresignedUrlExtension mockDelegateExtension = mock(AsyncPresignedUrlExtension.class); + // AsyncResponseTransformer mockTransformer = mock(AsyncResponseTransformer.class); + // AsyncResponseTransformer.SplitResult mockSplitResult = mock(AsyncResponseTransformer.SplitResult.class); + // PresignedUrlDownloadRequest req = PresignedUrlDownloadRequest.builder() + // .presignedUrl(new URL("https://s3.amazonaws.com/bucket/key?signature=abc")) + // .build(); + // when(mockDelegate.presignedUrlExtension()).thenReturn(mockDelegateExtension); + // when(mockTransformer.split(any(SplittingTransformerConfiguration.class))).thenReturn(mockSplitResult); + // when(mockSplitResult.publisher()).thenReturn(mock(software.amazon.awssdk.core.async.SdkPublisher.class)); + // S3AsyncClient s3AsyncClient = MultipartS3AsyncClient.create(mockDelegate, MultipartConfiguration.builder().build(), true); + // s3AsyncClient.presignedUrlExtension().getObject(req, mockTransformer); + // verify(mockTransformer, times(1)).split(any(SplittingTransformerConfiguration.class)); + // verify(mockDelegateExtension, never()).getObject(any(PresignedUrlDownloadRequest.class), any(AsyncResponseTransformer.class)); + // } + + @Test + void presignedUrlExtension_shouldReturnMultipartExtension() { + S3AsyncClient mockDelegate = mock(S3AsyncClient.class); + AsyncPresignedUrlExtension mockDelegateExtension = mock(AsyncPresignedUrlExtension.class); + when(mockDelegate.presignedUrlExtension()).thenReturn(mockDelegateExtension); + + S3AsyncClient s3AsyncClient = MultipartS3AsyncClient.create(mockDelegate, MultipartConfiguration.builder().build(), true); + AsyncPresignedUrlExtension extension = s3AsyncClient.presignedUrlExtension(); + + assertThat(extension).isInstanceOf(MultipartAsyncPresignedUrlExtension.class); + } } From 438322a4d53030070a98e064137f956b1c492dfa Mon Sep 17 00:00:00 2001 From: jencymaryjoseph <35571282+jencymaryjoseph@users.noreply.github.com> Date: Thu, 21 Aug 2025 10:30:35 -0700 Subject: [PATCH 11/30] Add S3 Transfer Manager presigned URL download request models (#6356) * Transfer Manager presigned URL download request models * Fixed checkstyle issues --- .../model/PresignedDownloadFileRequest.java | 286 +++++++++++++ .../s3/model/PresignedDownloadRequest.java | 391 ++++++++++++++++++ .../PresignedDownloadFileRequestTest.java | 119 ++++++ .../model/PresignedDownloadRequestTest.java | 136 ++++++ 4 files changed, 932 insertions(+) create mode 100644 services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/model/PresignedDownloadFileRequest.java create mode 100644 services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/model/PresignedDownloadRequest.java create mode 100644 services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/model/PresignedDownloadFileRequestTest.java create mode 100644 services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/model/PresignedDownloadRequestTest.java diff --git a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/model/PresignedDownloadFileRequest.java b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/model/PresignedDownloadFileRequest.java new file mode 100644 index 000000000000..b165292b22e5 --- /dev/null +++ b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/model/PresignedDownloadFileRequest.java @@ -0,0 +1,286 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.transfer.s3.model; + +import java.io.File; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.Objects; +import java.util.function.Consumer; +import software.amazon.awssdk.annotations.NotThreadSafe; +import software.amazon.awssdk.annotations.SdkPublicApi; +import software.amazon.awssdk.services.s3.presignedurl.model.PresignedUrlDownloadRequest; +import software.amazon.awssdk.transfer.s3.S3TransferManager; +import software.amazon.awssdk.transfer.s3.progress.TransferListener; +import software.amazon.awssdk.utils.ToString; +import software.amazon.awssdk.utils.Validate; +import software.amazon.awssdk.utils.builder.CopyableBuilder; +import software.amazon.awssdk.utils.builder.ToCopyableBuilder; + +/** + * Download an object using a pre-signed URL to a local file. For non-file-based downloads, you may use {@link + * PresignedDownloadRequest} instead. + * + * @see S3TransferManager#downloadFileWithPresignedUrl(PresignedDownloadFileRequest) + */ +@SdkPublicApi +public final class PresignedDownloadFileRequest + implements TransferObjectRequest, ToCopyableBuilder { + + private final Path destination; + private final PresignedUrlDownloadRequest presignedUrlDownloadRequest; + private final List transferListeners; + + private PresignedDownloadFileRequest(DefaultBuilder builder) { + this.destination = Validate.paramNotNull(builder.destination, "destination"); + this.presignedUrlDownloadRequest = Validate.paramNotNull(builder.presignedUrlDownloadRequest, + "presignedUrlDownloadRequest"); + this.transferListeners = builder.transferListeners; + } + + /** + * Creates a builder that can be used to create a {@link PresignedDownloadFileRequest}. + * + * @see S3TransferManager#downloadFileWithPresignedUrl(PresignedDownloadFileRequest) + */ + public static Builder builder() { + return new DefaultBuilder(); + } + + @Override + public Builder toBuilder() { + return new DefaultBuilder(this); + } + + /** + * The {@link Path} to file that response contents will be written to. The file must not exist or this method + * will throw an exception. If the file is not writable by the current user then an exception will be thrown. + * + * @return the destination path + */ + public Path destination() { + return destination; + } + + /** + * @return The {@link PresignedUrlDownloadRequest} request that should be used for the download + */ + public PresignedUrlDownloadRequest presignedUrlDownloadRequest() { + return presignedUrlDownloadRequest; + } + + /** + * + * @return List of {@link TransferListener}s that will be notified as part of this request. + */ + @Override + public List transferListeners() { + return transferListeners; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + + PresignedDownloadFileRequest that = (PresignedDownloadFileRequest) o; + + if (!Objects.equals(destination, that.destination)) { + return false; + } + if (!Objects.equals(presignedUrlDownloadRequest, that.presignedUrlDownloadRequest)) { + return false; + } + return Objects.equals(transferListeners, that.transferListeners); + } + + @Override + public int hashCode() { + int result = destination != null ? destination.hashCode() : 0; + result = 31 * result + (presignedUrlDownloadRequest != null ? presignedUrlDownloadRequest.hashCode() : 0); + result = 31 * result + (transferListeners != null ? transferListeners.hashCode() : 0); + return result; + } + + @Override + public String toString() { + return ToString.builder("PresignedDownloadFileRequest") + .add("destination", destination) + .add("presignedUrlDownloadRequest", presignedUrlDownloadRequest) + .add("transferListeners", transferListeners) + .build(); + } + + public static Class serializableBuilderClass() { + return DefaultBuilder.class; + } + + /** + * A builder for a {@link PresignedDownloadFileRequest}, created with {@link #builder()} + */ + @SdkPublicApi + @NotThreadSafe + public interface Builder extends CopyableBuilder { + + /** + * The {@link Path} to file that response contents will be written to. The file must not exist or this method + * will throw an exception. If the file is not writable by the current user then an exception will be thrown. + * + * @param destination the destination path + * @return Returns a reference to this object so that method calls can be chained together. + */ + Builder destination(Path destination); + + /** + * The file that response contents will be written to. The file must not exist or this method + * will throw an exception. If the file is not writable by the current user then an exception will be thrown. + * + * @param destination the destination path + * @return Returns a reference to this object so that method calls can be chained together. + */ + default Builder destination(File destination) { + Validate.paramNotNull(destination, "destination"); + return destination(destination.toPath()); + } + + /** + * The {@link PresignedUrlDownloadRequest} request that should be used for the download + * + * @param presignedUrlDownloadRequest the presigned URL download request + * @return a reference to this object so that method calls can be chained together. + * @see #presignedUrlDownloadRequest(Consumer) + */ + Builder presignedUrlDownloadRequest(PresignedUrlDownloadRequest presignedUrlDownloadRequest); + + /** + * The {@link PresignedUrlDownloadRequest} request that should be used for the download + * + *

+ * This is a convenience method that creates an instance of the {@link PresignedUrlDownloadRequest} builder avoiding the + * need to create one manually via {@link PresignedUrlDownloadRequest#builder()}. + * + * @param presignedUrlDownloadRequestBuilder the presigned URL download request + * @return a reference to this object so that method calls can be chained together. + * @see #presignedUrlDownloadRequest(PresignedUrlDownloadRequest) + */ + default Builder presignedUrlDownloadRequest( + Consumer presignedUrlDownloadRequestBuilder) { + PresignedUrlDownloadRequest request = PresignedUrlDownloadRequest.builder() + .applyMutation(presignedUrlDownloadRequestBuilder) + .build(); + presignedUrlDownloadRequest(request); + return this; + } + + /** + * The {@link TransferListener}s that will be notified as part of this request. This method overrides and replaces any + * transferListeners that have already been set. Add an optional request override configuration. + * + * @param transferListeners the collection of transferListeners + * @return Returns a reference to this object so that method calls can be chained together. + * @return This builder for method chaining. + * @see TransferListener + */ + Builder transferListeners(Collection transferListeners); + + /** + * Add a {@link TransferListener} that will be notified as part of this request. + * + * @param transferListener the transferListener to add + * @return Returns a reference to this object so that method calls can be chained together. + * @see TransferListener + */ + Builder addTransferListener(TransferListener transferListener); + + } + + private static final class DefaultBuilder implements Builder { + private Path destination; + private PresignedUrlDownloadRequest presignedUrlDownloadRequest; + private List transferListeners; + + private DefaultBuilder() { + } + + private DefaultBuilder(PresignedDownloadFileRequest presignedDownloadFileRequest) { + this.destination = presignedDownloadFileRequest.destination; + this.presignedUrlDownloadRequest = presignedDownloadFileRequest.presignedUrlDownloadRequest; + this.transferListeners = presignedDownloadFileRequest.transferListeners; + } + + @Override + public Builder destination(Path destination) { + this.destination = Validate.paramNotNull(destination, "destination"); + return this; + } + + public Path getDestination() { + return destination; + } + + public void setDestination(Path destination) { + destination(destination); + } + + @Override + public DefaultBuilder presignedUrlDownloadRequest(PresignedUrlDownloadRequest presignedUrlDownloadRequest) { + this.presignedUrlDownloadRequest = presignedUrlDownloadRequest; + return this; + } + + public PresignedUrlDownloadRequest getPresignedUrlDownloadRequest() { + return presignedUrlDownloadRequest; + } + + public void setPresignedUrlDownloadRequest(PresignedUrlDownloadRequest presignedUrlDownloadRequest) { + presignedUrlDownloadRequest(presignedUrlDownloadRequest); + } + + @Override + public DefaultBuilder transferListeners(Collection transferListeners) { + this.transferListeners = transferListeners != null ? new ArrayList<>(transferListeners) : null; + return this; + } + + @Override + public Builder addTransferListener(TransferListener transferListener) { + if (transferListeners == null) { + transferListeners = new ArrayList<>(); + } + transferListeners.add(transferListener); + return this; + } + + public List getTransferListeners() { + return transferListeners; + } + + public void setTransferListeners(Collection transferListeners) { + transferListeners(transferListeners); + } + + @Override + public PresignedDownloadFileRequest build() { + return new PresignedDownloadFileRequest(this); + } + } +} \ No newline at end of file diff --git a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/model/PresignedDownloadRequest.java b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/model/PresignedDownloadRequest.java new file mode 100644 index 000000000000..1a45eeb21254 --- /dev/null +++ b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/model/PresignedDownloadRequest.java @@ -0,0 +1,391 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.transfer.s3.model; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.Objects; +import java.util.function.Consumer; +import software.amazon.awssdk.annotations.SdkPublicApi; +import software.amazon.awssdk.core.async.AsyncResponseTransformer; +import software.amazon.awssdk.services.s3.model.GetObjectResponse; +import software.amazon.awssdk.services.s3.presignedurl.model.PresignedUrlDownloadRequest; +import software.amazon.awssdk.transfer.s3.S3TransferManager; +import software.amazon.awssdk.transfer.s3.model.PresignedDownloadRequest.TypedBuilder; +import software.amazon.awssdk.transfer.s3.progress.TransferListener; +import software.amazon.awssdk.utils.ToString; +import software.amazon.awssdk.utils.Validate; +import software.amazon.awssdk.utils.builder.CopyableBuilder; +import software.amazon.awssdk.utils.builder.ToCopyableBuilder; + +/** + * Represents the request to download an object using a pre-signed URL through the given + * {@link AsyncResponseTransformer}. For downloading to a file, + * you may use {@link PresignedDownloadFileRequest} instead. + * + * @see S3TransferManager#downloadWithPresignedUrl(PresignedDownloadRequest) + */ +@SdkPublicApi +public final class PresignedDownloadRequest + implements TransferObjectRequest, + ToCopyableBuilder, PresignedDownloadRequest> { + + private final AsyncResponseTransformer responseTransformer; + private final PresignedUrlDownloadRequest presignedUrlDownloadRequest; + private final List transferListeners; + + private PresignedDownloadRequest(DefaultTypedBuilder builder) { + this.responseTransformer = Validate.paramNotNull(builder.responseTransformer, "responseTransformer"); + this.presignedUrlDownloadRequest = Validate.paramNotNull(builder.presignedUrlDownloadRequest, + "presignedUrlDownloadRequest"); + this.transferListeners = builder.transferListeners; + } + + /** + * Creates a builder that can be used to create a {@link PresignedDownloadRequest}. + * + * @see UntypedBuilder + */ + public static UntypedBuilder builder() { + return new DefaultUntypedBuilder(); + } + + + @Override + public TypedBuilder toBuilder() { + return new DefaultTypedBuilder<>(this); + } + + /** + * The {@link AsyncResponseTransformer} that response contents will be written to. + * + * @return the response transformer + */ + public AsyncResponseTransformer responseTransformer() { + return responseTransformer; + } + + /** + * @return The {@link PresignedUrlDownloadRequest} request that should be used for the download + */ + public PresignedUrlDownloadRequest presignedUrlDownloadRequest() { + return presignedUrlDownloadRequest; + } + + /** + * @return the List of transferListeners. + * @see TypedBuilder#transferListeners(Collection) + */ + @Override + public List transferListeners() { + return transferListeners; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + + PresignedDownloadRequest that = (PresignedDownloadRequest) o; + + if (!Objects.equals(responseTransformer, that.responseTransformer)) { + return false; + } + if (!Objects.equals(presignedUrlDownloadRequest, that.presignedUrlDownloadRequest)) { + return false; + } + return Objects.equals(transferListeners, that.transferListeners); + } + + @Override + public int hashCode() { + int result = responseTransformer != null ? responseTransformer.hashCode() : 0; + result = 31 * result + (presignedUrlDownloadRequest != null ? presignedUrlDownloadRequest.hashCode() : 0); + result = 31 * result + (transferListeners != null ? transferListeners.hashCode() : 0); + return result; + } + + @Override + public String toString() { + return ToString.builder("PresignedDownloadRequest") + .add("responseTransformer", responseTransformer) + .add("presignedUrlDownloadRequest", presignedUrlDownloadRequest) + .add("transferListeners", transferListeners) + .build(); + } + + /** + * Initial calls to {@link PresignedDownloadRequest#builder()} return an {@link UntypedBuilder}, where the builder is not yet + * parameterized with the generic type associated with {@link PresignedDownloadRequest}. + * This prevents the otherwise awkward syntax of having to explicitly cast the builder type, e.g., + * 

+     * {@code PresignedDownloadRequest.>builder()}
+     *
+ * Instead, the type may be inferred as part of specifying the {@link #responseTransformer(AsyncResponseTransformer)} + * parameter, at which point the builder chain will return a new {@link TypedBuilder}. + */ + public interface UntypedBuilder { + + /** + * The {@link PresignedUrlDownloadRequest} request that should be used for the download + * + * @param presignedUrlDownloadRequest the presigned URL download request + * @return a reference to this object so that method calls can be chained together. + * @see #presignedUrlDownloadRequest(Consumer) + */ + UntypedBuilder presignedUrlDownloadRequest(PresignedUrlDownloadRequest presignedUrlDownloadRequest); + + /** + * The {@link PresignedUrlDownloadRequest} request that should be used for the download + *

+ * This is a convenience method that creates an instance of the {@link PresignedUrlDownloadRequest} + * builder, avoiding the need to create one manually via {@link PresignedUrlDownloadRequest#builder()}. + * + * @param presignedUrlDownloadRequestBuilder the presigned URL download request + * @return a reference to this object so that method calls can be chained together. + * @see #presignedUrlDownloadRequest(PresignedUrlDownloadRequest) + */ + default UntypedBuilder presignedUrlDownloadRequest( + Consumer presignedUrlDownloadRequestBuilder) { + PresignedUrlDownloadRequest request = PresignedUrlDownloadRequest.builder() + .applyMutation(presignedUrlDownloadRequestBuilder) + .build(); + presignedUrlDownloadRequest(request); + return this; + } + + /** + * The {@link TransferListener}s that will be notified as part of this request. This method + * overrides and replaces any transferListeners that have already been set. Add an optional + * request override configuration. + * + * @param transferListeners the collection of transferListeners + * @return Returns a reference to this object so that method calls can be chained together. + * @return This builder for method chaining. + * @see TransferListener + */ + UntypedBuilder transferListeners( + Collection transferListeners); + + /** + * Adds a {@link TransferListener} that will be notified as part of this request. + * + * @param transferListener the transferListener to add + * @return Returns a reference to this object so that method calls can be chained together. + * @see TransferListener + */ + UntypedBuilder addTransferListener(TransferListener transferListener); + + /** + * Specifies the {@link AsyncResponseTransformer} that should be used for the download. This + * method also infers the generic type of {@link PresignedDownloadRequest} to create, + * inferred from the second type parameter of the provided {@link AsyncResponseTransformer}. + * E.g, specifying {@link AsyncResponseTransformer#toBytes()} would result in inferring the + * type of the {@link PresignedDownloadRequest} to be of {@code ResponseBytes}. + * See the static factory methods available in {@link AsyncResponseTransformer}. + * + * @param responseTransformer the AsyncResponseTransformer + * @param the type of {@link PresignedDownloadRequest} to create + * @return a reference to this object so that method calls can be chained together. + * @see AsyncResponseTransformer + */ + TypedBuilder responseTransformer( + AsyncResponseTransformer responseTransformer); + } + + private static final class DefaultUntypedBuilder implements UntypedBuilder { + private PresignedUrlDownloadRequest presignedUrlDownloadRequest; + private List transferListeners; + + private DefaultUntypedBuilder() { + } + + @Override + public UntypedBuilder presignedUrlDownloadRequest(PresignedUrlDownloadRequest presignedUrlDownloadRequest) { + this.presignedUrlDownloadRequest = presignedUrlDownloadRequest; + return this; + } + + @Override + public UntypedBuilder transferListeners( + Collection transferListeners) { + this.transferListeners = transferListeners != null ? + new ArrayList<>(transferListeners) : null; + return this; + } + + @Override + public UntypedBuilder addTransferListener(TransferListener transferListener) { + if (transferListeners == null) { + transferListeners = new ArrayList<>(); + } + transferListeners.add(transferListener); + return this; + } + + public List getTransferListeners() { + return transferListeners; + } + + public void setTransferListeners( + Collection transferListeners) { + transferListeners(transferListeners); + } + + @Override + public TypedBuilder responseTransformer( + AsyncResponseTransformer responseTransformer) { + return new DefaultTypedBuilder() + .presignedUrlDownloadRequest(presignedUrlDownloadRequest) + .transferListeners(transferListeners) + .responseTransformer(responseTransformer); + } + } + + /** + * The type-parameterized version of {@link UntypedBuilder}. This builder's type is inferred as part of specifying {@link + * UntypedBuilder#responseTransformer(AsyncResponseTransformer)}, after which this builder can be used to construct a {@link + * PresignedDownloadRequest} with the same generic type. + */ + public interface TypedBuilder extends CopyableBuilder, PresignedDownloadRequest> { + + /** + * The {@link PresignedUrlDownloadRequest} request that should be used for the download + * + * @param presignedUrlDownloadRequest the presigned URL download request + * @return a reference to this object so that method calls can be chained together. + * @see #presignedUrlDownloadRequest(Consumer) + */ + TypedBuilder presignedUrlDownloadRequest(PresignedUrlDownloadRequest presignedUrlDownloadRequest); + + /** + * The {@link PresignedUrlDownloadRequest} request that should be used for the download + *

+ * This is a convenience method that creates an instance of the {@link PresignedUrlDownloadRequest} builder, + * avoiding the need to create one manually via {@link PresignedUrlDownloadRequest#builder()}. + * + * @param presignedUrlDownloadRequestBuilder the presigned URL download request + * @return a reference to this object so that method calls can be chained together. + * @see #presignedUrlDownloadRequest(PresignedUrlDownloadRequest) + */ + default TypedBuilder presignedUrlDownloadRequest( + Consumer presignedUrlDownloadRequestBuilder) { + PresignedUrlDownloadRequest request = PresignedUrlDownloadRequest.builder() + .applyMutation(presignedUrlDownloadRequestBuilder) + .build(); + presignedUrlDownloadRequest(request); + return this; + } + + /** + * The {@link TransferListener}s that will be notified as part of this request. This method overrides and + * replaces any transferListeners that have already been set. Add an optional request override + * configuration. + * + * @param transferListeners the collection of transferListeners + * @return Returns a reference to this object so that method calls can be chained together. + * @return This builder for method chaining. + * @see TransferListener + */ + TypedBuilder transferListeners( + Collection transferListeners); + + /** + * Add a {@link TransferListener} that will be notified as part of this request. + * + * @param transferListener the transferListener to add + * @return Returns a reference to this object so that method calls can be chained together. + * @see TransferListener + */ + TypedBuilder addTransferListener(TransferListener transferListener); + + /** + * Specifies the {@link AsyncResponseTransformer} that should be used for the download. The generic type used is + * constrained by the {@link UntypedBuilder#responseTransformer(AsyncResponseTransformer)} that was previously used to + * create this {@link TypedBuilder}. + * + * @param responseTransformer the AsyncResponseTransformer + * @return a reference to this object so that method calls can be chained together. + * @see AsyncResponseTransformer + */ + TypedBuilder responseTransformer( + AsyncResponseTransformer responseTransformer); + } + + private static class DefaultTypedBuilder implements TypedBuilder { + private PresignedUrlDownloadRequest presignedUrlDownloadRequest; + private List transferListeners; + private AsyncResponseTransformer responseTransformer; + + private DefaultTypedBuilder() { + } + + private DefaultTypedBuilder(PresignedDownloadRequest request) { + this.presignedUrlDownloadRequest = request.presignedUrlDownloadRequest; + this.responseTransformer = request.responseTransformer; + this.transferListeners = request.transferListeners; + } + + @Override + public TypedBuilder presignedUrlDownloadRequest(PresignedUrlDownloadRequest presignedUrlDownloadRequest) { + this.presignedUrlDownloadRequest = presignedUrlDownloadRequest; + return this; + } + + @Override + public TypedBuilder responseTransformer( + AsyncResponseTransformer responseTransformer) { + this.responseTransformer = responseTransformer; + return this; + } + + @Override + public TypedBuilder transferListeners( + Collection transferListeners) { + this.transferListeners = transferListeners != null ? + new ArrayList<>(transferListeners) : null; + return this; + } + + @Override + public TypedBuilder addTransferListener(TransferListener transferListener) { + if (transferListeners == null) { + transferListeners = new ArrayList<>(); + } + transferListeners.add(transferListener); + return this; + } + + public List getTransferListeners() { + return transferListeners; + } + + public void setTransferListeners( + Collection transferListeners) { + transferListeners(transferListeners); + } + + @Override + public PresignedDownloadRequest build() { + return new PresignedDownloadRequest<>(this); + } + } +} \ No newline at end of file diff --git a/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/model/PresignedDownloadFileRequestTest.java b/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/model/PresignedDownloadFileRequestTest.java new file mode 100644 index 000000000000..fb437e5f8796 --- /dev/null +++ b/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/model/PresignedDownloadFileRequestTest.java @@ -0,0 +1,119 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.transfer.s3.model; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import java.io.File; +import java.net.URL; +import java.nio.file.Path; +import java.nio.file.Paths; +import nl.jqno.equalsverifier.EqualsVerifier; +import org.junit.jupiter.api.Test; +import software.amazon.awssdk.services.s3.presignedurl.model.PresignedUrlDownloadRequest; + +class PresignedDownloadFileRequestTest { + + @Test + void build_withoutPresignedUrlDownloadRequest_throwsNullPointerException() { + assertThatThrownBy(() -> PresignedDownloadFileRequest.builder() + .destination(Paths.get(".")) + .build()) + .isInstanceOf(NullPointerException.class) + .hasMessageContaining("presignedUrlDownloadRequest"); + } + + @Test + void build_withoutDestination_throwsNullPointerException() { + assertThatThrownBy(() -> PresignedDownloadFileRequest.builder() + .presignedUrlDownloadRequest(createPresignedRequest()) + .build()) + .isInstanceOf(NullPointerException.class) + .hasMessageContaining("destination"); + } + + @Test + void build_withPath_setsDestinationCorrectly() { + Path path = Paths.get("."); + PresignedUrlDownloadRequest presignedRequest = createPresignedRequest(); + + PresignedDownloadFileRequest request = PresignedDownloadFileRequest.builder() + .destination(path) + .presignedUrlDownloadRequest(presignedRequest) + .build(); + + assertThat(request.destination()).isEqualTo(path); + assertThat(request.presignedUrlDownloadRequest()).isEqualTo(presignedRequest); + } + + @Test + void build_withFile_convertsToPathCorrectly() { + Path path = Paths.get("."); + PresignedUrlDownloadRequest presignedRequest = createPresignedRequest(); + + PresignedDownloadFileRequest request = PresignedDownloadFileRequest.builder() + .destination(path.toFile()) + .presignedUrlDownloadRequest(presignedRequest) + .build(); + + assertThat(request.destination()).isEqualTo(path); + } + + @Test + void destination_withNullFile_throwsNullPointerException() { + File file = null; + + assertThatThrownBy(() -> PresignedDownloadFileRequest.builder() + .destination(file)) + .isInstanceOf(NullPointerException.class) + .hasMessageContaining("destination"); + } + + @Test + void presignedUrlDownloadRequest_withConsumerBuilder_buildsCorrectly() { + Path path = Paths.get("."); + + PresignedDownloadFileRequest request = PresignedDownloadFileRequest.builder() + .destination(path) + .presignedUrlDownloadRequest(b -> b.presignedUrl(createTestUrl())) + .build(); + + assertThat(request.presignedUrlDownloadRequest()).isNotNull(); + assertThat(request.presignedUrlDownloadRequest().presignedUrl()).isEqualTo(createTestUrl()); + } + + @Test + void equalsHashCodeTest() { + EqualsVerifier.forClass(PresignedDownloadFileRequest.class) + .withNonnullFields("destination", "presignedUrlDownloadRequest") + .verify(); + } + + private PresignedUrlDownloadRequest createPresignedRequest() { + return PresignedUrlDownloadRequest.builder() + .presignedUrl(createTestUrl()) + .build(); + } + + private URL createTestUrl() { + try { + return new URL("https://example.com/test"); + } catch (Exception e) { + throw new RuntimeException(e); + } + } +} \ No newline at end of file diff --git a/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/model/PresignedDownloadRequestTest.java b/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/model/PresignedDownloadRequestTest.java new file mode 100644 index 000000000000..15381bfd50b6 --- /dev/null +++ b/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/model/PresignedDownloadRequestTest.java @@ -0,0 +1,136 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.transfer.s3.model; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import java.net.URL; +import nl.jqno.equalsverifier.EqualsVerifier; +import org.junit.jupiter.api.Test; +import software.amazon.awssdk.core.ResponseBytes; +import software.amazon.awssdk.core.async.AsyncResponseTransformer; +import software.amazon.awssdk.services.s3.model.GetObjectResponse; +import software.amazon.awssdk.services.s3.presignedurl.model.PresignedUrlDownloadRequest; + +class PresignedDownloadRequestTest { + + @Test + void build_withoutPresignedUrlDownloadRequest_throwsNullPointerException() { + assertThatThrownBy(() -> PresignedDownloadRequest.builder() + .responseTransformer(AsyncResponseTransformer.toBytes()) + .build()) + .isInstanceOf(NullPointerException.class) + .hasMessageContaining("presignedUrlDownloadRequest"); + } + + @Test + void builder_withoutResponseTransformer_returnsUntypedBuilder() { + PresignedDownloadRequest.UntypedBuilder untypedBuilder = PresignedDownloadRequest.builder() + .presignedUrlDownloadRequest(createPresignedRequest()); + + assertThat(untypedBuilder).isNotNull(); + } + + @Test + void build_withResponseTransformer_createsRequestCorrectly() { + AsyncResponseTransformer> responseTransformer = + AsyncResponseTransformer.toBytes(); + PresignedUrlDownloadRequest presignedRequest = createPresignedRequest(); + + PresignedDownloadRequest> request = + PresignedDownloadRequest.builder() + .presignedUrlDownloadRequest(presignedRequest) + .responseTransformer(responseTransformer) + .build(); + + assertThat(request.responseTransformer()).isEqualTo(responseTransformer); + assertThat(request.presignedUrlDownloadRequest()).isEqualTo(presignedRequest); + } + + @Test + void presignedUrlDownloadRequest_withConsumerBuilder_buildsCorrectly() { + AsyncResponseTransformer> responseTransformer = + AsyncResponseTransformer.toBytes(); + + PresignedDownloadRequest> request = + PresignedDownloadRequest.builder() + .presignedUrlDownloadRequest(b -> b.presignedUrl(createTestUrl())) + .responseTransformer(responseTransformer) + .build(); + + assertThat(request.presignedUrlDownloadRequest()).isNotNull(); + assertThat(request.presignedUrlDownloadRequest().presignedUrl()).isEqualTo(createTestUrl()); + } + + @Test + void responseTransformer_withNull_throwsNullPointerException() { + assertThatThrownBy(() -> PresignedDownloadRequest.builder() + .presignedUrlDownloadRequest(createPresignedRequest()) + .responseTransformer(null) + .build()) + .isInstanceOf(NullPointerException.class) + .hasMessageContaining("responseTransformer"); + } + + @Test + void builder_untypedToTypedTransition_worksCorrectly() { + PresignedDownloadRequest.UntypedBuilder untypedBuilder = PresignedDownloadRequest.builder() + .presignedUrlDownloadRequest(createPresignedRequest()); + + PresignedDownloadRequest> request = + untypedBuilder.responseTransformer(AsyncResponseTransformer.toBytes()) + .build(); + + assertThat(request.responseTransformer()).isNotNull(); + assertThat(request.presignedUrlDownloadRequest()).isNotNull(); + } + + @Test + void toBuilder_copiesAllFields() { + PresignedDownloadRequest> original = + PresignedDownloadRequest.builder() + .presignedUrlDownloadRequest(createPresignedRequest()) + .responseTransformer(AsyncResponseTransformer.toBytes()) + .build(); + + PresignedDownloadRequest> copy = original.toBuilder().build(); + + assertThat(copy.responseTransformer()).isEqualTo(original.responseTransformer()); + assertThat(copy.presignedUrlDownloadRequest()).isEqualTo(original.presignedUrlDownloadRequest()); + } + + @Test + void equalsHashCodeTest() { + EqualsVerifier.forClass(PresignedDownloadRequest.class) + .withNonnullFields("responseTransformer", "presignedUrlDownloadRequest") + .verify(); + } + + private PresignedUrlDownloadRequest createPresignedRequest() { + return PresignedUrlDownloadRequest.builder() + .presignedUrl(createTestUrl()) + .build(); + } + + private URL createTestUrl() { + try { + return new URL("https://example.com/test"); + } catch (Exception e) { + throw new RuntimeException(e); + } + } +} \ No newline at end of file From 6e31588dc8cb81e0d5043212d0e0d66dd62dcc3b Mon Sep 17 00:00:00 2001 From: jencymaryjoseph <35571282+jencymaryjoseph@users.noreply.github.com> Date: Thu, 21 Aug 2025 11:22:23 -0700 Subject: [PATCH 12/30] Add Range-based Multipart download Subscriber for Pre-signed URLs (#6331) * Range based multipart download subscriber * Refactor code and add tests * Fixed sonarcube issue * Address PR comments: use Long type, inline variables, allow empty files * Refactored subscriber and tests * delete testutil * Added validation tests * Fixed failing tests for IfMatch header * Refactor: split validatePartAndRequestMore into separate methods * Fixed validation tests * Remove onComplete after subscription cancel * Make validatePart boolean function --- .../multipart/MultipartDownloadUtils.java | 53 +++ .../multipart/PresignedUrlDownloadHelper.java | 31 +- ...ignedUrlMultipartDownloaderSubscriber.java | 276 +++++++++++++ .../DefaultAsyncPresignedUrlExtension.java | 1 + .../PresignedUrlDownloadRequestWrapper.java | 26 +- .../model/PresignedUrlDownloadRequest.java | 34 +- .../multipart/MultipartDownloadUtilsTest.java | 17 + ...lMultipartDownloaderSubscriberTckTest.java | 153 ++++++++ ...ipartDownloaderSubscriberWiremockTest.java | 371 ++++++++++++++++++ ...resignedUrlDownloadRequestWrapperTest.java | 44 ++- 10 files changed, 983 insertions(+), 23 deletions(-) create mode 100644 services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/PresignedUrlMultipartDownloaderSubscriber.java create mode 100644 services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/multipart/PresignedUrlMultipartDownloaderSubscriberTckTest.java create mode 100644 services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/multipart/PresignedUrlMultipartDownloaderSubscriberWiremockTest.java diff --git a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/MultipartDownloadUtils.java b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/MultipartDownloadUtils.java index 807b6a8bbbc0..8be5bb8ee5b4 100644 --- a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/MultipartDownloadUtils.java +++ b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/MultipartDownloadUtils.java @@ -20,12 +20,17 @@ import java.util.Collections; import java.util.List; import java.util.Optional; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import software.amazon.awssdk.annotations.SdkInternalApi; import software.amazon.awssdk.services.s3.model.GetObjectRequest; +import software.amazon.awssdk.services.s3.model.S3Request; @SdkInternalApi public final class MultipartDownloadUtils { + private static final Pattern CONTENT_RANGE_PATTERN = Pattern.compile("bytes\\s+(\\d+)-(\\d+)/(\\d+)"); + private MultipartDownloadUtils() { } @@ -58,4 +63,52 @@ public static Optional multipartDownloadResumeCo .flatMap(conf -> Optional.ofNullable(conf.executionAttributes().getAttribute(MULTIPART_DOWNLOAD_RESUME_CONTEXT))); } + /** + * This method checks the + * {@link software.amazon.awssdk.services.s3.multipart.S3MultipartExecutionAttribute#MULTIPART_DOWNLOAD_RESUME_CONTEXT} + * execution attributes for a context object and returns it if it finds one. Otherwise, returns an empty Optional. + * + * @param request the request to look for execution attributes + * @return the MultipartDownloadResumeContext if one is found, otherwise an empty Optional. + */ + public static Optional multipartDownloadResumeContext(S3Request request) { + return request + .overrideConfiguration() + .flatMap(conf -> Optional.ofNullable(conf.executionAttributes().getAttribute(MULTIPART_DOWNLOAD_RESUME_CONTEXT))); + } + + /** + * Parses the start byte from a Content-Range header. + * + * @param contentRange the Content-Range header value (e.g., "bytes 0-1023/2048") + * @return the start byte position, or -1 if parsing fails + */ + public static long parseStartByteFromContentRange(String contentRange) { + if (contentRange == null) { + return -1; + } + Matcher matcher = CONTENT_RANGE_PATTERN.matcher(contentRange); + if (!matcher.matches()) { + return -1; + } + return Long.parseLong(matcher.group(1)); + } + + /** + * Parses the total size from a Content-Range header. + * + * @param contentRange the Content-Range header value (e.g., "bytes 0-1023/2048") + * @return the total size, or empty if parsing fails + */ + public static Optional parseContentRangeForTotalSize(String contentRange) { + if (contentRange == null) { + return Optional.empty(); + } + Matcher matcher = CONTENT_RANGE_PATTERN.matcher(contentRange); + if (!matcher.matches()) { + return Optional.empty(); + } + return Optional.of(Long.parseLong(matcher.group(3))); + } + } diff --git a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/PresignedUrlDownloadHelper.java b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/PresignedUrlDownloadHelper.java index cfb94e8a0242..7ee520b76c23 100644 --- a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/PresignedUrlDownloadHelper.java +++ b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/PresignedUrlDownloadHelper.java @@ -19,6 +19,7 @@ import software.amazon.awssdk.annotations.SdkInternalApi; import software.amazon.awssdk.core.SplittingTransformerConfiguration; import software.amazon.awssdk.core.async.AsyncResponseTransformer; +import software.amazon.awssdk.core.exception.SdkClientException; import software.amazon.awssdk.services.s3.S3AsyncClient; import software.amazon.awssdk.services.s3.model.GetObjectResponse; import software.amazon.awssdk.services.s3.presignedurl.AsyncPresignedUrlExtension; @@ -63,15 +64,25 @@ public CompletableFuture downloadObject( .build(); AsyncResponseTransformer.SplitResult split = asyncResponseTransformer.split(splittingConfig); - // TODO: PresignedUrlMultipartDownloaderSubscriber needs to be implemented in next PR - // PresignedUrlMultipartDownloaderSubscriber subscriber = - // new PresignedUrlMultipartDownloaderSubscriber( - // s3AsyncClient, - // presignedRequest, - // configuredPartSizeInBytes); - // - // split.publisher().subscribe(subscriber); - // return split.resultFuture(); - throw new UnsupportedOperationException("Multipart presigned URL download not yet implemented - TODO in next PR"); + PresignedUrlMultipartDownloaderSubscriber subscriber = + new PresignedUrlMultipartDownloaderSubscriber( + s3AsyncClient, + presignedRequest, + configuredPartSizeInBytes); + + split.publisher().subscribe(subscriber); + return split.resultFuture(); + } + + static SdkClientException invalidContentRangeHeader(String contentRange) { + return SdkClientException.create("Invalid Content-Range header: " + contentRange); + } + + static SdkClientException missingContentRangeHeader() { + return SdkClientException.create("No Content-Range header in response"); + } + + static SdkClientException invalidContentLength() { + return SdkClientException.create("Invalid or missing Content-Length in response"); } } diff --git a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/PresignedUrlMultipartDownloaderSubscriber.java b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/PresignedUrlMultipartDownloaderSubscriber.java new file mode 100644 index 000000000000..975ae0339e78 --- /dev/null +++ b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/PresignedUrlMultipartDownloaderSubscriber.java @@ -0,0 +1,276 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.services.s3.internal.multipart; + +import java.util.Optional; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.atomic.AtomicInteger; +import org.reactivestreams.Subscriber; +import org.reactivestreams.Subscription; +import software.amazon.awssdk.annotations.Immutable; +import software.amazon.awssdk.annotations.SdkInternalApi; +import software.amazon.awssdk.annotations.ThreadSafe; +import software.amazon.awssdk.core.async.AsyncResponseTransformer; +import software.amazon.awssdk.core.exception.SdkClientException; +import software.amazon.awssdk.services.s3.S3AsyncClient; +import software.amazon.awssdk.services.s3.model.GetObjectResponse; +import software.amazon.awssdk.services.s3.presignedurl.model.PresignedUrlDownloadRequest; +import software.amazon.awssdk.utils.Logger; + +/** + * A subscriber implementation that will download all individual parts for a multipart presigned URL download request. + * It receives individual {@link AsyncResponseTransformer} instances which will be used to perform the individual + * range-based part requests using presigned URLs. This is a 'one-shot' class, it should NOT be reused + * for more than one multipart download. + * + *

Unlike the standard {@link MultipartDownloaderSubscriber} which uses S3's native multipart API with part numbers, + * this subscriber uses HTTP range requests against presigned URLs to achieve multipart download functionality. + *

This implementation is thread-safe and handles concurrent part downloads while maintaining proper + * ordering and validation of responses.

+ */ +@ThreadSafe +@Immutable +@SdkInternalApi +public class PresignedUrlMultipartDownloaderSubscriber + implements Subscriber> { + + private static final Logger log = Logger.loggerFor(PresignedUrlMultipartDownloaderSubscriber.class); + private static final String BYTES_RANGE_PREFIX = "bytes="; + + private final S3AsyncClient s3AsyncClient; + private final PresignedUrlDownloadRequest presignedUrlDownloadRequest; + private final Long configuredPartSizeInBytes; + private final CompletableFuture future; + private final Object lock = new Object(); + private final AtomicInteger completedParts; + private final AtomicInteger requestsSent; + + private volatile Long totalContentLength; + private volatile Integer totalParts; + private volatile String eTag; + private Subscription subscription; + + public PresignedUrlMultipartDownloaderSubscriber( + S3AsyncClient s3AsyncClient, + PresignedUrlDownloadRequest presignedUrlDownloadRequest, + long configuredPartSizeInBytes) { + this.s3AsyncClient = s3AsyncClient; + this.presignedUrlDownloadRequest = presignedUrlDownloadRequest; + this.configuredPartSizeInBytes = configuredPartSizeInBytes; + this.completedParts = new AtomicInteger(0); + this.requestsSent = new AtomicInteger(0); + this.future = new CompletableFuture<>(); + } + + @Override + public void onSubscribe(Subscription s) { + if (subscription != null) { + s.cancel(); + return; + } + this.subscription = s; + s.request(1); + } + + @Override + public void onNext(AsyncResponseTransformer asyncResponseTransformer) { + if (asyncResponseTransformer == null) { + throw new NullPointerException("onNext must not be called with null asyncResponseTransformer"); + } + + int nextPartIndex; + synchronized (lock) { + nextPartIndex = completedParts.get(); + if (totalParts != null && nextPartIndex >= totalParts) { + log.debug(() -> String.format("Completing multipart download after a total of %d parts downloaded.", totalParts)); + subscription.cancel(); + return; + } + completedParts.incrementAndGet(); + } + makeRangeRequest(nextPartIndex, asyncResponseTransformer); + } + + private void makeRangeRequest(int partIndex, + AsyncResponseTransformer asyncResponseTransformer) { + PresignedUrlDownloadRequest partRequest = createRangedGetRequest(partIndex); + log.debug(() -> "Sending range request for part " + partIndex + " with range=" + partRequest.range()); + + requestsSent.incrementAndGet(); + s3AsyncClient.presignedUrlExtension() + .getObject(partRequest, asyncResponseTransformer) + .whenComplete((response, error) -> { + if (error != null) { + log.debug(() -> "Error encountered during part request for part " + partIndex); + handleError(error); + return; + } + if (validatePart(response, partIndex, asyncResponseTransformer)) { + requestMoreIfNeeded(completedParts.get()); + } + }); + } + + private boolean validatePart(GetObjectResponse response, int partIndex, + AsyncResponseTransformer asyncResponseTransformer) { + int totalComplete = completedParts.get(); + log.debug(() -> String.format("Completed part %d", totalComplete)); + + String responseETag = response.eTag(); + String responseContentRange = response.contentRange(); + if (eTag == null) { + this.eTag = responseETag; + log.debug(() -> String.format("Multipart object ETag: %s", this.eTag)); + } + + Optional validationError = validateResponse(response, partIndex); + if (validationError.isPresent()) { + log.debug(() -> "Response validation failed", validationError.get()); + asyncResponseTransformer.exceptionOccurred(validationError.get()); + handleError(validationError.get()); + return false; + } + + if (totalContentLength == null && responseContentRange != null) { + Optional parsedContentLength = MultipartDownloadUtils.parseContentRangeForTotalSize(responseContentRange); + if (!parsedContentLength.isPresent()) { + SdkClientException error = PresignedUrlDownloadHelper.invalidContentRangeHeader(responseContentRange); + log.debug(() -> "Failed to parse content range", error); + asyncResponseTransformer.exceptionOccurred(error); + handleError(error); + return false; + } + + this.totalContentLength = parsedContentLength.get(); + this.totalParts = calculateTotalParts(totalContentLength, configuredPartSizeInBytes); + log.debug(() -> String.format("Total content length: %d, Total parts: %d", totalContentLength, totalParts)); + } + return true; + } + + private void requestMoreIfNeeded(int totalComplete) { + synchronized (lock) { + if (hasMoreParts(totalComplete)) { + subscription.request(1); + } else { + if (totalParts != null && requestsSent.get() != totalParts) { + handleError(new IllegalStateException( + "Request count mismatch. Expected: " + totalParts + ", sent: " + requestsSent.get())); + return; + } + log.debug(() -> String.format("Completing multipart download after a total of %d parts downloaded.", totalParts)); + subscription.cancel(); + } + } + } + + private Optional validateResponse(GetObjectResponse response, int partIndex) { + if (response == null) { + return Optional.of(SdkClientException.create("Response cannot be null")); + } + String contentRange = response.contentRange(); + if (contentRange == null) { + return Optional.of(PresignedUrlDownloadHelper.missingContentRangeHeader()); + } + + Long contentLength = response.contentLength(); + if (contentLength == null || contentLength < 0) { + return Optional.of(PresignedUrlDownloadHelper.invalidContentLength()); + } + + long expectedStartByte = partIndex * configuredPartSizeInBytes; + long expectedEndByte; + if (totalContentLength != null) { + expectedEndByte = Math.min(expectedStartByte + configuredPartSizeInBytes - 1, totalContentLength - 1); + } else { + expectedEndByte = expectedStartByte + configuredPartSizeInBytes - 1; + } + String expectedRange = "bytes " + expectedStartByte + "-" + expectedEndByte + "/"; + if (!contentRange.startsWith(expectedRange)) { + return Optional.of(SdkClientException.create( + "Content-Range mismatch. Expected range starting with: " + expectedRange + + ", but got: " + contentRange)); + } + + long expectedPartSize; + if (totalContentLength != null && partIndex == totalParts - 1) { + expectedPartSize = totalContentLength - (partIndex * configuredPartSizeInBytes); + } else { + expectedPartSize = configuredPartSizeInBytes; + } + if (!contentLength.equals(expectedPartSize)) { + return Optional.of(SdkClientException.create( + String.format("Part content length validation failed for part %d. Expected: %d, but got: %d", + partIndex, expectedPartSize, contentLength))); + } + + long actualStartByte = MultipartDownloadUtils.parseStartByteFromContentRange(contentRange); + if (actualStartByte != expectedStartByte) { + return Optional.of(SdkClientException.create( + "Content range offset mismatch for part " + partIndex + + ". Expected start: " + expectedStartByte + ", but got: " + actualStartByte)); + } + + return Optional.empty(); + } + + private int calculateTotalParts(long contentLength, long partSize) { + return (int) Math.ceil((double) contentLength / partSize); + } + + private boolean hasMoreParts(int completedPartsCount) { + return totalParts != null && completedPartsCount < totalParts; + } + + private PresignedUrlDownloadRequest createRangedGetRequest(int partIndex) { + long startByte = partIndex * configuredPartSizeInBytes; + long endByte; + if (totalContentLength != null) { + endByte = Math.min(startByte + configuredPartSizeInBytes - 1, totalContentLength - 1); + } else { + endByte = startByte + configuredPartSizeInBytes - 1; + } + String rangeHeader = BYTES_RANGE_PREFIX + startByte + "-" + endByte; + PresignedUrlDownloadRequest.Builder builder = presignedUrlDownloadRequest.toBuilder() + .range(rangeHeader); + if (partIndex > 0 && eTag != null) { + builder.ifMatch(eTag); + } + return builder.build(); + } + + private void handleError(Throwable t) { + synchronized (lock) { + if (subscription != null) { + subscription.cancel(); + } + } + onError(t); + } + + @Override + public void onError(Throwable t) { + log.debug(() -> "Error in multipart download", t); + future.completeExceptionally(t); + } + + @Override + public void onComplete() { + future.complete(null); + } + +} \ No newline at end of file diff --git a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/presignedurl/DefaultAsyncPresignedUrlExtension.java b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/presignedurl/DefaultAsyncPresignedUrlExtension.java index db7515b24b54..d640482c5c36 100644 --- a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/presignedurl/DefaultAsyncPresignedUrlExtension.java +++ b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/presignedurl/DefaultAsyncPresignedUrlExtension.java @@ -90,6 +90,7 @@ public CompletableFuture getObject( PresignedUrlDownloadRequestWrapper internalRequest = PresignedUrlDownloadRequestWrapper.builder() .url(presignedUrlDownloadRequest.presignedUrl()) .range(presignedUrlDownloadRequest.range()) + .ifMatch(presignedUrlDownloadRequest.ifMatch()) .build(); MetricCollector apiCallMetricCollector = metricPublishers.isEmpty() ? diff --git a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/presignedurl/model/PresignedUrlDownloadRequestWrapper.java b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/presignedurl/model/PresignedUrlDownloadRequestWrapper.java index 983c6b9c13eb..5d5569523967 100644 --- a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/presignedurl/model/PresignedUrlDownloadRequestWrapper.java +++ b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/presignedurl/model/PresignedUrlDownloadRequestWrapper.java @@ -48,18 +48,27 @@ public final class PresignedUrlDownloadRequestWrapper extends S3Request { .traits(LocationTrait.builder().location(MarshallLocation.HEADER).locationName("Range") .unmarshallLocationName("Range").build()).build(); + private static final SdkField IF_MATCH_FIELD = SdkField + .builder(MarshallingType.STRING) + .memberName("IfMatch") + .getter(getter(PresignedUrlDownloadRequestWrapper::ifMatch)) + .traits(LocationTrait.builder().location(MarshallLocation.HEADER).locationName("If-Match") + .unmarshallLocationName("If-Match").build()).build(); + private static final List> SDK_FIELDS = Collections.unmodifiableList( - Arrays.asList(RANGE_FIELD)); + Arrays.asList(RANGE_FIELD, IF_MATCH_FIELD)); private static final Map> SDK_NAME_TO_FIELD = memberNameToFieldInitializer(); private final URL url; private final String range; + private final String ifMatch; private PresignedUrlDownloadRequestWrapper(Builder builder) { super(builder); this.url = builder.url; this.range = builder.range; + this.ifMatch = builder.ifMatch; } public URL url() { @@ -70,6 +79,10 @@ public String range() { return range; } + public String ifMatch() { + return ifMatch; + } + @Override public List> sdkFields() { return SDK_FIELDS; @@ -87,6 +100,7 @@ private static Function getter(Function> memberNameToFieldInitializer() { Map> map = new HashMap<>(); map.put("Range", RANGE_FIELD); + map.put("IfMatch", IF_MATCH_FIELD); return Collections.unmodifiableMap(map); } @@ -111,7 +125,7 @@ public boolean equals(Object obj) { return false; } PresignedUrlDownloadRequestWrapper that = (PresignedUrlDownloadRequestWrapper) obj; - return Objects.equals(url, that.url) && Objects.equals(range, that.range); + return Objects.equals(url, that.url) && Objects.equals(range, that.range) && Objects.equals(ifMatch, that.ifMatch); } @Override @@ -119,12 +133,14 @@ public int hashCode() { int result = Objects.hashCode(super.hashCode()); result = 31 * result + Objects.hashCode(url); result = 31 * result + Objects.hashCode(range); + result = 31 * result + Objects.hashCode(ifMatch); return result; } public static final class Builder extends S3Request.BuilderImpl { private URL url; private String range; + private String ifMatch; public Builder() { } @@ -133,6 +149,7 @@ public Builder() { super(request); this.url = request.url(); this.range = request.range(); + this.ifMatch = request.ifMatch(); } public Builder url(URL url) { @@ -145,6 +162,11 @@ public Builder range(String range) { return this; } + public Builder ifMatch(String ifMatch) { + this.ifMatch = ifMatch; + return this; + } + @Override public PresignedUrlDownloadRequestWrapper build() { return new PresignedUrlDownloadRequestWrapper(this); diff --git a/services/s3/src/main/java/software/amazon/awssdk/services/s3/presignedurl/model/PresignedUrlDownloadRequest.java b/services/s3/src/main/java/software/amazon/awssdk/services/s3/presignedurl/model/PresignedUrlDownloadRequest.java index 380c234b82c8..6c19159251fc 100644 --- a/services/s3/src/main/java/software/amazon/awssdk/services/s3/presignedurl/model/PresignedUrlDownloadRequest.java +++ b/services/s3/src/main/java/software/amazon/awssdk/services/s3/presignedurl/model/PresignedUrlDownloadRequest.java @@ -31,10 +31,12 @@ public final class PresignedUrlDownloadRequest implements ToCopyableBuilder { private final URL presignedUrl; private final String range; + private final String ifMatch; private PresignedUrlDownloadRequest(BuilderImpl builder) { this.presignedUrl = builder.presignedUrl; this.range = builder.range; + this.ifMatch = builder.ifMatch; } /** @@ -65,6 +67,18 @@ public String range() { return range; } + /** + *

+ * Return the object only if its entity tag (ETag) is the same as the one specified in this header, + * otherwise return a 412 (precondition failed) error. + *

+ * + * @return The If-Match header value, or null if not specified. + */ + public String ifMatch() { + return ifMatch; + } + @Override public Builder toBuilder() { return new BuilderImpl(this); @@ -83,6 +97,7 @@ public int hashCode() { int hashCode = 1; hashCode = 31 * hashCode + Objects.hashCode(presignedUrl()); hashCode = 31 * hashCode + Objects.hashCode(range()); + hashCode = 31 * hashCode + Objects.hashCode(ifMatch()); return hashCode; } @@ -96,7 +111,8 @@ public boolean equals(Object obj) { } PresignedUrlDownloadRequest other = (PresignedUrlDownloadRequest) obj; return Objects.equals(presignedUrl(), other.presignedUrl()) && - Objects.equals(range(), other.range()); + Objects.equals(range(), other.range()) && + Objects.equals(ifMatch(), other.ifMatch()); } @Override @@ -104,6 +120,7 @@ public String toString() { return ToString.builder("PresignedUrlDownloadRequest") .add("PresignedUrl", presignedUrl()) .add("Range", range()) + .add("IfMatch", ifMatch()) .build(); } @@ -121,11 +138,19 @@ public interface Builder extends CopyableBuilder> { + + private S3AsyncClient s3mock; + + public PresignedUrlMultipartDownloaderSubscriberTckTest() { + super(new TestEnvironment()); + this.s3mock = Mockito.mock(S3AsyncClient.class); + } + + @Override + public Subscriber> + createSubscriber(WhiteboxSubscriberProbe> probe) { + AsyncPresignedUrlExtension presignedUrlExtension = Mockito.mock(AsyncPresignedUrlExtension.class); + when(s3mock.presignedUrlExtension()).thenReturn(presignedUrlExtension); + + CompletableFuture firstPartResponse = CompletableFuture.completedFuture( + GetObjectResponse.builder() + .contentRange("bytes 0-8388607/33554432") + .contentLength(8388608L) // 8MB + .eTag("\"test-etag-12345\"") + .build() + ); + + CompletableFuture subsequentPartResponse = CompletableFuture.completedFuture( + GetObjectResponse.builder() + .contentRange("bytes 8388608-16777215/33554432") + .contentLength(8388608L) // 8MB + .eTag("\"test-etag-12345\"") + .build() + ); + + when(presignedUrlExtension.getObject(any(PresignedUrlDownloadRequest.class), any(AsyncResponseTransformer.class))) + .thenReturn(firstPartResponse) + .thenReturn(subsequentPartResponse) + .thenReturn(subsequentPartResponse) + .thenReturn(subsequentPartResponse); + + return new PresignedUrlMultipartDownloaderSubscriber( + s3mock, + createTestPresignedUrlRequest(), + 8 * 1024 * 1024L + ) { + @Override + public void onError(Throwable throwable) { + super.onError(throwable); + probe.registerOnError(throwable); + } + + @Override + public void onSubscribe(Subscription subscription) { + super.onSubscribe(subscription); + probe.registerOnSubscribe(new SubscriberPuppet() { + @Override + public void triggerRequest(long elements) { + subscription.request(elements); + } + + @Override + public void signalCancel() { + subscription.cancel(); + } + }); + } + + @Override + public void onNext(AsyncResponseTransformer item) { + super.onNext(item); + probe.registerOnNext(item); + } + + @Override + public void onComplete() { + super.onComplete(); + probe.registerOnComplete(); + } + }; + } + + @Override + public AsyncResponseTransformer createElement(int element) { + return new TestAsyncResponseTransformer(); + } + + private PresignedUrlDownloadRequest createTestPresignedUrlRequest() { + try { + return PresignedUrlDownloadRequest.builder() + .presignedUrl(java.net.URI.create("https://test-bucket.s3.amazonaws.com/test-key").toURL()) + .build(); + } catch (MalformedURLException e) { + throw new RuntimeException("Failed to create test URL", e); + } + } + + private static class TestAsyncResponseTransformer implements AsyncResponseTransformer { + private CompletableFuture future; + + @Override + public CompletableFuture prepare() { + this.future = new CompletableFuture<>(); + return this.future; + } + + @Override + public void onResponse(GetObjectResponse response) { + this.future.complete(response); + } + + @Override + public void onStream(SdkPublisher publisher) { + } + + @Override + public void exceptionOccurred(Throwable error) { + future.completeExceptionally(error); + } + } +} diff --git a/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/multipart/PresignedUrlMultipartDownloaderSubscriberWiremockTest.java b/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/multipart/PresignedUrlMultipartDownloaderSubscriberWiremockTest.java new file mode 100644 index 000000000000..3228ec2b8800 --- /dev/null +++ b/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/multipart/PresignedUrlMultipartDownloaderSubscriberWiremockTest.java @@ -0,0 +1,371 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.services.s3.internal.multipart; + +import static com.github.tomakehurst.wiremock.client.WireMock.aResponse; +import static com.github.tomakehurst.wiremock.client.WireMock.get; +import static com.github.tomakehurst.wiremock.client.WireMock.stubFor; +import static com.github.tomakehurst.wiremock.client.WireMock.urlEqualTo; +import static com.github.tomakehurst.wiremock.client.WireMock.matching; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.junit.jupiter.api.Assertions.assertArrayEquals; + +import com.github.tomakehurst.wiremock.junit5.WireMockRuntimeInfo; +import com.github.tomakehurst.wiremock.junit5.WireMockTest; +import java.io.IOException; +import java.net.MalformedURLException; +import java.net.URI; +import java.net.URL; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Arrays; +import java.util.UUID; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import software.amazon.awssdk.core.async.AsyncResponseTransformer; +import software.amazon.awssdk.core.exception.SdkClientException; +import software.amazon.awssdk.services.s3.S3AsyncClient; +import software.amazon.awssdk.services.s3.model.S3Exception; +import software.amazon.awssdk.services.s3.multipart.MultipartConfiguration; +import software.amazon.awssdk.services.s3.presignedurl.model.PresignedUrlDownloadRequest; + +@WireMockTest +class PresignedUrlMultipartDownloaderSubscriberWiremockTest { + + private static final String PRESIGNED_URL_PATH = "/presigned-url"; + private static final byte[] TEST_DATA = "This is exactly a 32 byte string".getBytes(StandardCharsets.UTF_8); + + private S3AsyncClient s3AsyncClient; + private String presignedUrlBase; + private URL presignedUrl; + private Path tempFile; + + @BeforeEach + public void setup(WireMockRuntimeInfo wiremock) throws MalformedURLException { + MultipartConfiguration multipartConfig = MultipartConfiguration.builder() + .minimumPartSizeInBytes(16L) + .build(); + s3AsyncClient = S3AsyncClient.builder() + .endpointOverride(URI.create("http://localhost:" + wiremock.getHttpPort())) + .multipartEnabled(true) + .multipartConfiguration(multipartConfig) + .build(); + presignedUrlBase = "http://localhost:" + wiremock.getHttpPort(); + presignedUrl = createPresignedUrl(); + } + + @Test + void presignedUrlDownload_withMultipartData_shouldReceiveCompleteBody() { + stubSuccessfulPresignedUrlResponse(); + byte[] result = s3AsyncClient.presignedUrlExtension() + .getObject(PresignedUrlDownloadRequest.builder() + .presignedUrl(presignedUrl) + .build(), + AsyncResponseTransformer.toBytes()) + .join() + .asByteArray(); + assertArrayEquals(TEST_DATA, result); + } + + @Test + void presignedUrlDownload_withRangeHeader_shouldReceivePartialContent() { + stubSuccessfulRangeResponse(); + byte[] result = s3AsyncClient.presignedUrlExtension() + .getObject(PresignedUrlDownloadRequest.builder() + .presignedUrl(presignedUrl) + .range("bytes=0-10") + .build(), + AsyncResponseTransformer.toBytes()) + .join() + .asByteArray(); + byte[] expectedPartial = Arrays.copyOfRange(TEST_DATA, 0, 11); + assertArrayEquals(expectedPartial, result); + } + + @Test + void presignedUrlDownload_whenRequestFails_shouldThrowException() { + stubFailedPresignedUrlResponse(); + assertThatThrownBy(() -> s3AsyncClient.presignedUrlExtension() + .getObject(PresignedUrlDownloadRequest.builder() + .presignedUrl(presignedUrl) + .build(), + AsyncResponseTransformer.toBytes()) + .join()) + .hasRootCauseInstanceOf(S3Exception.class); + } + + @Test + void presignedUrlDownload_withFileTransformer_shouldWork() throws IOException { + stubSuccessfulPresignedUrlResponse(); + tempFile = createUniqueTempFile(); + s3AsyncClient.presignedUrlExtension() + .getObject(PresignedUrlDownloadRequest.builder() + .presignedUrl(presignedUrl) + .build(), + AsyncResponseTransformer.toFile(tempFile)) + .join(); + assertThat(tempFile.toFile()).exists(); + assertThat(tempFile.toFile().length()).isGreaterThan(0); + } + + @Test + void presignedUrlDownload_whenFirstRequestFails_shouldThrowException() { + stubInternalServerError(); + assertThatThrownBy(() -> s3AsyncClient.presignedUrlExtension() + .getObject(PresignedUrlDownloadRequest.builder() + .presignedUrl(presignedUrl) + .build(), + AsyncResponseTransformer.toBytes()) + .join()) + .hasRootCauseInstanceOf(S3Exception.class); + } + + @Test + void presignedUrlDownload_whenSecondRequestFails_shouldThrowException() { + stubPartialFailureScenario(); + assertThatThrownBy(() -> s3AsyncClient.presignedUrlExtension() + .getObject(PresignedUrlDownloadRequest.builder() + .presignedUrl(presignedUrl) + .build(), + AsyncResponseTransformer.toBytes()) + .join()) + .hasRootCauseInstanceOf(S3Exception.class); + } + + @Test + void presignedUrlDownload_whenIOErrorOccurs_shouldThrowException() { + stubConnectionReset(); + assertThatThrownBy(() -> s3AsyncClient.presignedUrlExtension() + .getObject(PresignedUrlDownloadRequest.builder() + .presignedUrl(presignedUrl) + .build(), + AsyncResponseTransformer.toBytes()) + .join()) + .hasCauseInstanceOf(IOException.class); + } + + + @Test + void onNext_withNullTransformer_shouldThrowException() { + PresignedUrlMultipartDownloaderSubscriber subscriber = createTestSubscriber(); + + assertThatThrownBy(() -> subscriber.onNext(null)) + .isInstanceOf(NullPointerException.class) + .hasMessageContaining("onNext must not be called with null asyncResponseTransformer"); + } + + @Test + void presignedUrlDownload_withMissingContentRange_shouldFailRequest() { + stubResponseWithMissingContentRange(); + assertThatThrownBy(() -> s3AsyncClient.presignedUrlExtension() + .getObject(PresignedUrlDownloadRequest.builder() + .presignedUrl(presignedUrl) + .build(), + AsyncResponseTransformer.toBytes()) + .join()) + .hasRootCauseInstanceOf(SdkClientException.class) + .hasMessageContaining("No Content-Range header in response"); + } + + @Test + void presignedUrlDownload_withInvalidContentLength_shouldFailRequest() { + stubResponseWithInvalidContentLength(); + assertThatThrownBy(() -> s3AsyncClient.presignedUrlExtension() + .getObject(PresignedUrlDownloadRequest.builder() + .presignedUrl(presignedUrl) + .build(), + AsyncResponseTransformer.toBytes()) + .join()) + .hasRootCauseInstanceOf(SdkClientException.class) + .hasMessageContaining("Invalid or missing Content-Length in response"); + } + + @Test + void presignedUrlDownload_withContentRangeMismatch_shouldFailRequest() { + stubResponseWithContentRangeMismatch(); + assertThatThrownBy(() -> s3AsyncClient.presignedUrlExtension() + .getObject(PresignedUrlDownloadRequest.builder() + .presignedUrl(presignedUrl) + .build(), + AsyncResponseTransformer.toBytes()) + .join()) + .hasRootCauseInstanceOf(SdkClientException.class) + .hasMessageContaining("Content-Range mismatch"); + } + + @Test + void presignedUrlDownload_withContentLengthMismatch_shouldFailRequest() { + stubResponseWithContentLengthMismatch(); + assertThatThrownBy(() -> s3AsyncClient.presignedUrlExtension() + .getObject(PresignedUrlDownloadRequest.builder() + .presignedUrl(presignedUrl) + .build(), + AsyncResponseTransformer.toBytes()) + .join()) + .hasRootCauseInstanceOf(SdkClientException.class) + .hasMessageContaining("Part content length validation failed"); + } + + @AfterEach + void cleanup() { + if (tempFile != null && Files.exists(tempFile)) { + try { + Files.delete(tempFile); + } catch (IOException e) { + } + } + } + + private static Path createUniqueTempFile() throws IOException { + String uniqueName = "test-" + UUID.randomUUID().toString(); + Path tempFile = Files.createTempFile(uniqueName, ".tmp"); + Files.deleteIfExists(tempFile); + return tempFile; + } + + private void stubSuccessfulPresignedUrlResponse() { + // Stub for first part (bytes 0-15) + stubFor(get(urlEqualTo(PRESIGNED_URL_PATH)) + .withHeader("Range", matching("bytes=0-15")) + .willReturn(aResponse() + .withStatus(206) + .withHeader("Content-Type", "application/octet-stream") + .withHeader("Content-Length", "16") + .withHeader("Content-Range", "bytes 0-15/32") + .withHeader("ETag", "\"test-etag\"") + .withBody(Arrays.copyOfRange(TEST_DATA, 0, 16)))); + + // Stub for second part (bytes 16-31) + stubFor(get(urlEqualTo(PRESIGNED_URL_PATH)) + .withHeader("Range", matching("bytes=16-31")) + .willReturn(aResponse() + .withStatus(206) + .withHeader("Content-Type", "application/octet-stream") + .withHeader("Content-Length", "16") + .withHeader("Content-Range", "bytes 16-31/32") + .withHeader("ETag", "\"test-etag\"") + .withBody(Arrays.copyOfRange(TEST_DATA, 16, 32)))); + } + + private void stubSuccessfulRangeResponse() { + byte[] partialData = Arrays.copyOfRange(TEST_DATA, 0, 11); + stubFor(get(urlEqualTo(PRESIGNED_URL_PATH)) + .willReturn(aResponse() + .withStatus(206) + .withHeader("Content-Type", "application/octet-stream") + .withHeader("Content-Length", String.valueOf(partialData.length)) + .withHeader("Content-Range", "bytes 0-10/" + TEST_DATA.length) + .withHeader("ETag", "\"test-etag\"") + .withBody(partialData))); + } + + private void stubFailedPresignedUrlResponse() { + stubFor(get(urlEqualTo(PRESIGNED_URL_PATH)) + .willReturn(aResponse() + .withStatus(404) + .withBody("NoSuchKeyThe specified key does not exist."))); + } + + private void stubInternalServerError() { + stubFor(get(urlEqualTo(PRESIGNED_URL_PATH)) + .willReturn(aResponse() + .withStatus(500) + .withBody("InternalErrorInternal Server Error"))); + } + + private void stubPartialFailureScenario() { + stubFor(get(urlEqualTo(PRESIGNED_URL_PATH)) + .inScenario("partial-failure") + .whenScenarioStateIs("Started") + .willReturn(aResponse() + .withStatus(206) + .withHeader("Content-Type", "application/octet-stream") + .withHeader("Content-Length", "16") + .withHeader("Content-Range", "bytes 0-15/" + TEST_DATA.length) + .withHeader("ETag", "\"test-etag\"") + .withBody(Arrays.copyOfRange(TEST_DATA, 0, 16))) + .willSetStateTo("first-success")); + + stubFor(get(urlEqualTo(PRESIGNED_URL_PATH)) + .inScenario("partial-failure") + .whenScenarioStateIs("first-success") + .willReturn(aResponse() + .withStatus(500) + .withBody("InternalErrorSecond request failed"))); + } + + private void stubConnectionReset() { + stubFor(get(urlEqualTo(PRESIGNED_URL_PATH)) + .willReturn(aResponse() + .withFault(com.github.tomakehurst.wiremock.http.Fault.CONNECTION_RESET_BY_PEER))); + } + + private URL createPresignedUrl() throws MalformedURLException { + return new URL(presignedUrlBase + PRESIGNED_URL_PATH); + } + + private PresignedUrlMultipartDownloaderSubscriber createTestSubscriber() { + return new PresignedUrlMultipartDownloaderSubscriber( + s3AsyncClient, + PresignedUrlDownloadRequest.builder().presignedUrl(presignedUrl).build(), + 1024L); + } + + private void stubResponseWithMissingContentRange() { + stubFor(get(urlEqualTo(PRESIGNED_URL_PATH)) + .willReturn(aResponse() + .withStatus(206) + .withHeader("Content-Type", "application/octet-stream") + .withHeader("Content-Length", "16") + .withHeader("ETag", "\"test-etag\"") + .withBody(Arrays.copyOfRange(TEST_DATA, 0, 16)))); + } + + private void stubResponseWithInvalidContentLength() { + stubFor(get(urlEqualTo(PRESIGNED_URL_PATH)) + .willReturn(aResponse() + .withStatus(206) + .withHeader("Content-Type", "application/octet-stream") + .withHeader("Content-Range", "bytes 0-15/" + TEST_DATA.length) + .withHeader("ETag", "\"test-etag\"") + .withBody(Arrays.copyOfRange(TEST_DATA, 0, 16)))); + } + + private void stubResponseWithContentRangeMismatch() { + stubFor(get(urlEqualTo(PRESIGNED_URL_PATH)) + .willReturn(aResponse() + .withStatus(206) + .withHeader("Content-Type", "application/octet-stream") + .withHeader("Content-Length", "16") + .withHeader("Content-Range", "bytes 5000-5015/" + TEST_DATA.length) + .withHeader("ETag", "\"test-etag\"") + .withBody(Arrays.copyOfRange(TEST_DATA, 0, 16)))); + } + + private void stubResponseWithContentLengthMismatch() { + stubFor(get(urlEqualTo(PRESIGNED_URL_PATH)) + .willReturn(aResponse() + .withStatus(206) + .withHeader("Content-Type", "application/octet-stream") + .withHeader("Content-Length", "8") + .withHeader("Content-Range", "bytes 0-15/" + TEST_DATA.length) + .withHeader("ETag", "\"test-etag\"") + .withBody(Arrays.copyOfRange(TEST_DATA, 0, 8)))); + } +} \ No newline at end of file diff --git a/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/presignedurl/model/PresignedUrlDownloadRequestWrapperTest.java b/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/presignedurl/model/PresignedUrlDownloadRequestWrapperTest.java index c1280667eb26..8b301a332199 100644 --- a/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/presignedurl/model/PresignedUrlDownloadRequestWrapperTest.java +++ b/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/presignedurl/model/PresignedUrlDownloadRequestWrapperTest.java @@ -39,10 +39,12 @@ void basicProperties_shouldWork() throws Exception { PresignedUrlDownloadRequestWrapper request = PresignedUrlDownloadRequestWrapper.builder() .url(new URL("https://example.com")) .range("bytes=0-100") + .ifMatch("\"etag-123\"") .build(); assertThat(request.url()).isEqualTo(new URL("https://example.com")); assertThat(request.range()).isEqualTo("bytes=0-100"); + assertThat(request.ifMatch()).isEqualTo("\"etag-123\""); } @Test @@ -54,12 +56,15 @@ void sdkFields_shouldReturnExpectedFields() throws Exception { List> fields = request.sdkFields(); - assertThat(fields).hasSize(1); - assertThat(fields.get(0).memberName()).isEqualTo("Range"); - assertThat(fields.get(0).location()).isEqualTo(MarshallLocation.HEADER); - - // Assert that range value is present - Object rangeValue = fields.get(0).getValueOrDefault(request); + assertThat(fields).hasSize(2); + assertThat(fields).extracting(SdkField::memberName) + .containsExactlyInAnyOrder("Range", "IfMatch"); + assertThat(fields).allMatch(field -> field.location() == MarshallLocation.HEADER); + SdkField rangeField = fields.stream() + .filter(f -> "Range".equals(f.memberName())) + .findFirst() + .orElseThrow(() -> new AssertionError("Range field not found")); + Object rangeValue = rangeField.getValueOrDefault(request); assertThat(rangeValue).isNotNull(); assertThat(rangeValue).isEqualTo("bytes=0-100"); } @@ -73,9 +78,10 @@ void sdkFieldNameToField_shouldReturnExpectedMapping() throws Exception { Map> fieldMap = request.sdkFieldNameToField(); assertThat(fieldMap) - .hasSize(1) - .containsKey("Range"); + .hasSize(2) + .containsKeys("Range", "IfMatch"); assertThat(fieldMap.get("Range").memberName()).isEqualTo("Range"); + assertThat(fieldMap.get("IfMatch").memberName()).isEqualTo("IfMatch"); } @Test @@ -85,10 +91,28 @@ void rangeField_shouldMarshalCorrectly() throws Exception { .range("bytes=0-1023") .build(); - // Test that the SdkField can extract the range value - SdkField rangeField = request.sdkFields().get(0); + SdkField rangeField = request.sdkFields().stream() + .filter(f -> "Range".equals(f.memberName())) + .findFirst() + .orElseThrow(() -> new AssertionError("Range field not found")); Object extractedValue = rangeField.getValueOrDefault(request); assertThat(extractedValue).isEqualTo("bytes=0-1023"); } + + @Test + void ifMatchField_shouldMarshalCorrectly() throws Exception { + PresignedUrlDownloadRequestWrapper request = PresignedUrlDownloadRequestWrapper.builder() + .url(new URL("https://example.com")) + .ifMatch("\"etag-value\"") + .build(); + + SdkField ifMatchField = request.sdkFields().stream() + .filter(f -> "IfMatch".equals(f.memberName())) + .findFirst() + .orElseThrow(() -> new AssertionError("IfMatch field not found")); + Object extractedValue = ifMatchField.getValueOrDefault(request); + + assertThat(extractedValue).isEqualTo("\"etag-value\""); + } } From 150039220b16742bbc8a1e4f1ca1e8bfb8eddafb Mon Sep 17 00:00:00 2001 From: jencymaryjoseph <35571282+jencymaryjoseph@users.noreply.github.com> Date: Mon, 4 May 2026 22:44:40 -0700 Subject: [PATCH 13/30] Fix multipart presigned URL download: parallel support and error propagation --- ...ignedUrlMultipartDownloaderSubscriber.java | 335 ++++++++++++++++++ .../multipart/PresignedUrlDownloadHelper.java | 31 +- ...ignedUrlMultipartDownloaderSubscriber.java | 68 ++-- ...lMultipartDownloaderSubscriberTckTest.java | 134 +++++++ ...dUrlMultipartDownloaderSubscriberTest.java | 250 +++++++++++++ ...lMultipartDownloaderSubscriberTckTest.java | 3 +- ...ipartDownloaderSubscriberWiremockTest.java | 269 ++++++++------ 7 files changed, 943 insertions(+), 147 deletions(-) create mode 100644 services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/ParallelPresignedUrlMultipartDownloaderSubscriber.java create mode 100644 services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/multipart/ParallelPresignedUrlMultipartDownloaderSubscriberTckTest.java create mode 100644 services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/multipart/ParallelPresignedUrlMultipartDownloaderSubscriberTest.java diff --git a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/ParallelPresignedUrlMultipartDownloaderSubscriber.java b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/ParallelPresignedUrlMultipartDownloaderSubscriber.java new file mode 100644 index 000000000000..9c8f77ada905 --- /dev/null +++ b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/ParallelPresignedUrlMultipartDownloaderSubscriber.java @@ -0,0 +1,335 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.services.s3.internal.multipart; + +import java.util.Map; +import java.util.Optional; +import java.util.Queue; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import org.reactivestreams.Subscriber; +import org.reactivestreams.Subscription; +import software.amazon.awssdk.annotations.SdkInternalApi; +import software.amazon.awssdk.core.async.AsyncResponseTransformer; +import software.amazon.awssdk.core.exception.SdkClientException; +import software.amazon.awssdk.services.s3.S3AsyncClient; +import software.amazon.awssdk.services.s3.model.GetObjectResponse; +import software.amazon.awssdk.services.s3.presignedurl.model.PresignedUrlDownloadRequest; +import software.amazon.awssdk.utils.CompletableFutureUtils; +import software.amazon.awssdk.utils.Logger; +import software.amazon.awssdk.utils.Pair; + +/** + * A parallel subscriber for multipart presigned URL downloads that writes parts concurrently. + * Used with {@link software.amazon.awssdk.core.internal.async.FileAsyncResponseTransformerPublisher} + * when {@code parallelSplitSupported() == true} (i.e., toFile() downloads). + * + *

Unlike {@link PresignedUrlMultipartDownloaderSubscriber} which requests one part at a time, + * this subscriber requests up to {@code maxInFlightParts} concurrently, similar to + * {@link ParallelMultipartDownloaderSubscriber} for regular multipart downloads.

+ */ +@SdkInternalApi +public class ParallelPresignedUrlMultipartDownloaderSubscriber + implements Subscriber> { + + private static final Logger log = Logger.loggerFor(ParallelPresignedUrlMultipartDownloaderSubscriber.class); + private static final String BYTES_RANGE_PREFIX = "bytes="; + + private final S3AsyncClient s3AsyncClient; + private final PresignedUrlDownloadRequest presignedUrlDownloadRequest; + private final long configuredPartSizeInBytes; + private final CompletableFuture resultFuture; + private final int maxInFlightParts; + + private final AtomicInteger partNumber = new AtomicInteger(0); + private final AtomicInteger completedParts = new AtomicInteger(0); + private final AtomicInteger inFlightRequestsNum = new AtomicInteger(0); + private final AtomicBoolean isCompletedExceptionally = new AtomicBoolean(false); + private final AtomicBoolean processingPending = new AtomicBoolean(false); + private final Map> inFlightRequests = new ConcurrentHashMap<>(); + private final Queue>> pendingTransformers = + new ConcurrentLinkedQueue<>(); + + private final Object subscriptionLock = new Object(); + private Subscription subscription; + + private volatile Long totalContentLength; + private volatile Integer totalParts; + private volatile String eTag; + private volatile GetObjectResponse firstResponse; + + public ParallelPresignedUrlMultipartDownloaderSubscriber( + S3AsyncClient s3AsyncClient, + PresignedUrlDownloadRequest presignedUrlDownloadRequest, + long configuredPartSizeInBytes, + CompletableFuture resultFuture, + int maxInFlightParts) { + this.s3AsyncClient = s3AsyncClient; + this.presignedUrlDownloadRequest = presignedUrlDownloadRequest; + this.configuredPartSizeInBytes = configuredPartSizeInBytes; + this.resultFuture = resultFuture; + this.maxInFlightParts = maxInFlightParts; + } + + @Override + public void onSubscribe(Subscription s) { + if (this.subscription != null) { + s.cancel(); + return; + } + this.subscription = s; + s.request(1); + } + + @Override + public void onNext(AsyncResponseTransformer asyncResponseTransformer) { + if (asyncResponseTransformer == null) { + throw new NullPointerException("onNext must not be called with null asyncResponseTransformer"); + } + + int currentPart = partNumber.getAndIncrement(); + + if (currentPart == 0) { + sendFirstRequest(asyncResponseTransformer); + } else { + if (totalParts != null && currentPart >= totalParts) { + return; + } + if (totalParts != null) { + processRequest(asyncResponseTransformer, currentPart); + } else { + pendingTransformers.offer(Pair.of(currentPart, asyncResponseTransformer)); + } + } + } + + private void sendFirstRequest(AsyncResponseTransformer transformer) { + PresignedUrlDownloadRequest partRequest = createRangedGetRequest(0); + log.debug(() -> "Sending first range request with range=" + partRequest.range()); + + CompletableFuture response = + s3AsyncClient.presignedUrlExtension().getObject(partRequest, transformer); + + inFlightRequests.put(0, response); + inFlightRequestsNum.incrementAndGet(); + CompletableFutureUtils.forwardExceptionTo(resultFuture, response); + + response.whenComplete((res, error) -> { + if (error != null || isCompletedExceptionally.get()) { + handlePartError(error, 0); + return; + } + + inFlightRequests.remove(0); + inFlightRequestsNum.decrementAndGet(); + completedParts.incrementAndGet(); + + // Discover size and ETag from first response + this.eTag = res.eTag(); + this.firstResponse = res; + + String contentRange = res.contentRange(); + if (contentRange == null) { + handlePartError(PresignedUrlDownloadHelper.missingContentRangeHeader(), 0); + return; + } + + Optional parsedTotal = MultipartDownloadUtils.parseContentRangeForTotalSize(contentRange); + if (!parsedTotal.isPresent()) { + handlePartError(PresignedUrlDownloadHelper.invalidContentRangeHeader(contentRange), 0); + return; + } + + this.totalContentLength = parsedTotal.get(); + this.totalParts = calculateTotalParts(totalContentLength, configuredPartSizeInBytes); + log.debug(() -> String.format("Total content length: %d, Total parts: %d", totalContentLength, totalParts)); + + if (totalParts <= 1) { + synchronized (subscriptionLock) { + subscription.cancel(); + } + resultFuture.complete(firstResponse); + return; + } + + processPendingTransformers(); + + int remainingParts = totalParts - 1; + int toRequest = Math.min(remainingParts, maxInFlightParts); + synchronized (subscriptionLock) { + subscription.request(toRequest); + } + }); + } + + private void processRequest(AsyncResponseTransformer transformer, + int currentPart) { + if (currentPart >= totalParts) { + return; + } + + if (inFlightRequestsNum.get() >= maxInFlightParts) { + pendingTransformers.offer(Pair.of(currentPart, transformer)); + return; + } + + sendPartRequest(transformer, currentPart); + processPendingTransformers(); + } + + private void sendPartRequest(AsyncResponseTransformer transformer, + int partIndex) { + if (isCompletedExceptionally.get()) { + return; + } + + PresignedUrlDownloadRequest partRequest = createRangedGetRequest(partIndex); + log.debug(() -> "Sending range request for part " + partIndex + " with range=" + partRequest.range()); + + CompletableFuture response = + s3AsyncClient.presignedUrlExtension().getObject(partRequest, transformer); + + inFlightRequests.put(partIndex, response); + inFlightRequestsNum.incrementAndGet(); + CompletableFutureUtils.forwardExceptionTo(resultFuture, response); + + response.whenComplete((res, error) -> { + if (error != null || isCompletedExceptionally.get()) { + handlePartError(error, partIndex); + return; + } + + Optional validationError = validatePartResponse(res, partIndex); + if (validationError.isPresent()) { + handlePartError(validationError.get(), partIndex); + return; + } + + log.debug(() -> "Completed part: " + partIndex); + inFlightRequests.remove(partIndex); + inFlightRequestsNum.decrementAndGet(); + int totalComplete = completedParts.incrementAndGet(); + + if (totalComplete >= totalParts) { + synchronized (subscriptionLock) { + subscription.cancel(); + } + resultFuture.complete(firstResponse); + } else { + processPendingTransformers(); + synchronized (subscriptionLock) { + subscription.request(1); + } + } + }); + } + + private void processPendingTransformers() { + do { + if (!processingPending.compareAndSet(false, true)) { + return; + } + try { + while (!pendingTransformers.isEmpty() && inFlightRequestsNum.get() < maxInFlightParts) { + Pair> pair = + pendingTransformers.poll(); + if (pair != null && pair.left() < totalParts) { + sendPartRequest(pair.right(), pair.left()); + } + } + } finally { + processingPending.set(false); + } + } while (!pendingTransformers.isEmpty() && inFlightRequestsNum.get() < maxInFlightParts); + } + + private Optional validatePartResponse(GetObjectResponse response, int partIndex) { + String contentRange = response.contentRange(); + if (contentRange == null) { + return Optional.of(PresignedUrlDownloadHelper.missingContentRangeHeader()); + } + Long contentLength = response.contentLength(); + if (contentLength == null || contentLength < 0) { + return Optional.of(PresignedUrlDownloadHelper.invalidContentLength()); + } + long expectedStartByte = partIndex * configuredPartSizeInBytes; + long expectedEndByte = Math.min(expectedStartByte + configuredPartSizeInBytes - 1, totalContentLength - 1); + String expectedRange = "bytes " + expectedStartByte + "-" + expectedEndByte + "/"; + if (!contentRange.startsWith(expectedRange)) { + return Optional.of(SdkClientException.create( + "Content-Range mismatch. Expected range starting with: " + expectedRange + + ", but got: " + contentRange)); + } + long expectedPartSize = (partIndex == totalParts - 1) + ? totalContentLength - (partIndex * configuredPartSizeInBytes) + : configuredPartSizeInBytes; + if (!contentLength.equals(expectedPartSize)) { + return Optional.of(SdkClientException.create( + String.format("Part content length validation failed for part %d. Expected: %d, but got: %d", + partIndex, expectedPartSize, contentLength))); + } + return Optional.empty(); + } + + private void handlePartError(Throwable error, int partIndex) { + if (isCompletedExceptionally.compareAndSet(false, true)) { + log.debug(() -> "Error on part " + partIndex, error); + resultFuture.completeExceptionally(error); + inFlightRequests.values().forEach(future -> future.cancel(true)); + synchronized (subscriptionLock) { + if (subscription != null) { + subscription.cancel(); + } + } + } + } + + private PresignedUrlDownloadRequest createRangedGetRequest(int partIndex) { + long startByte = partIndex * configuredPartSizeInBytes; + long endByte; + if (totalContentLength != null) { + endByte = Math.min(startByte + configuredPartSizeInBytes - 1, totalContentLength - 1); + } else { + endByte = startByte + configuredPartSizeInBytes - 1; + } + String rangeHeader = BYTES_RANGE_PREFIX + startByte + "-" + endByte; + PresignedUrlDownloadRequest.Builder builder = presignedUrlDownloadRequest.toBuilder() + .range(rangeHeader); + if (partIndex > 0 && eTag != null) { + builder.ifMatch(eTag); + } + return builder.build(); + } + + private int calculateTotalParts(long contentLength, long partSize) { + return (int) Math.ceil((double) contentLength / partSize); + } + + @Override + public void onError(Throwable t) { + log.debug(() -> "Error in parallel multipart download", t); + resultFuture.completeExceptionally(t); + inFlightRequests.values().forEach(future -> future.cancel(true)); + } + + @Override + public void onComplete() { + // Completion is handled by resultFuture + } +} diff --git a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/PresignedUrlDownloadHelper.java b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/PresignedUrlDownloadHelper.java index 7ee520b76c23..f62ec1d7d202 100644 --- a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/PresignedUrlDownloadHelper.java +++ b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/PresignedUrlDownloadHelper.java @@ -30,6 +30,7 @@ @SdkInternalApi public class PresignedUrlDownloadHelper { private static final Logger log = Logger.loggerFor(PresignedUrlDownloadHelper.class); + private static final int DEFAULT_MAX_IN_FLIGHT_PARTS = 10; private final S3AsyncClient s3AsyncClient; private final AsyncPresignedUrlExtension asyncPresignedUrlExtension; @@ -64,12 +65,38 @@ public CompletableFuture downloadObject( .build(); AsyncResponseTransformer.SplitResult split = asyncResponseTransformer.split(splittingConfig); + + if (split.parallelSplitSupported()) { + return downloadPartsInParallel(presignedRequest, split); + } + return downloadPartsSerially(presignedRequest, split); + } + + private CompletableFuture downloadPartsInParallel( + PresignedUrlDownloadRequest presignedRequest, + AsyncResponseTransformer.SplitResult split) { + log.debug(() -> "Using parallel multipart download for presigned URL"); + ParallelPresignedUrlMultipartDownloaderSubscriber subscriber = + new ParallelPresignedUrlMultipartDownloaderSubscriber( + s3AsyncClient, + presignedRequest, + configuredPartSizeInBytes, + (CompletableFuture) split.resultFuture(), + DEFAULT_MAX_IN_FLIGHT_PARTS); + split.publisher().subscribe(subscriber); + return split.resultFuture(); + } + + private CompletableFuture downloadPartsSerially( + PresignedUrlDownloadRequest presignedRequest, + AsyncResponseTransformer.SplitResult split) { + log.debug(() -> "Using serial multipart download for presigned URL"); PresignedUrlMultipartDownloaderSubscriber subscriber = new PresignedUrlMultipartDownloaderSubscriber( s3AsyncClient, presignedRequest, - configuredPartSizeInBytes); - + configuredPartSizeInBytes, + split.resultFuture()); split.publisher().subscribe(subscriber); return split.resultFuture(); } diff --git a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/PresignedUrlMultipartDownloaderSubscriber.java b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/PresignedUrlMultipartDownloaderSubscriber.java index 975ae0339e78..7c042ef5aefa 100644 --- a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/PresignedUrlMultipartDownloaderSubscriber.java +++ b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/PresignedUrlMultipartDownloaderSubscriber.java @@ -54,6 +54,7 @@ public class PresignedUrlMultipartDownloaderSubscriber private final PresignedUrlDownloadRequest presignedUrlDownloadRequest; private final Long configuredPartSizeInBytes; private final CompletableFuture future; + private final CompletableFuture resultFuture; private final Object lock = new Object(); private final AtomicInteger completedParts; private final AtomicInteger requestsSent; @@ -66,13 +67,15 @@ public class PresignedUrlMultipartDownloaderSubscriber public PresignedUrlMultipartDownloaderSubscriber( S3AsyncClient s3AsyncClient, PresignedUrlDownloadRequest presignedUrlDownloadRequest, - long configuredPartSizeInBytes) { + long configuredPartSizeInBytes, + CompletableFuture resultFuture) { this.s3AsyncClient = s3AsyncClient; this.presignedUrlDownloadRequest = presignedUrlDownloadRequest; this.configuredPartSizeInBytes = configuredPartSizeInBytes; this.completedParts = new AtomicInteger(0); this.requestsSent = new AtomicInteger(0); this.future = new CompletableFuture<>(); + this.resultFuture = resultFuture; } @Override @@ -109,20 +112,20 @@ private void makeRangeRequest(int partIndex, GetObjectResponse> asyncResponseTransformer) { PresignedUrlDownloadRequest partRequest = createRangedGetRequest(partIndex); log.debug(() -> "Sending range request for part " + partIndex + " with range=" + partRequest.range()); - + requestsSent.incrementAndGet(); s3AsyncClient.presignedUrlExtension() - .getObject(partRequest, asyncResponseTransformer) - .whenComplete((response, error) -> { - if (error != null) { - log.debug(() -> "Error encountered during part request for part " + partIndex); - handleError(error); - return; - } - if (validatePart(response, partIndex, asyncResponseTransformer)) { - requestMoreIfNeeded(completedParts.get()); - } - }); + .getObject(partRequest, asyncResponseTransformer) + .whenComplete((response, error) -> { + if (error != null) { + log.debug(() -> "Error encountered during part request for part " + partIndex); + handleError(error); + return; + } + if (validatePart(response, partIndex, asyncResponseTransformer)) { + requestMoreIfNeeded(completedParts.get()); + } + }); } private boolean validatePart(GetObjectResponse response, int partIndex, @@ -137,14 +140,6 @@ private boolean validatePart(GetObjectResponse response, int partIndex, log.debug(() -> String.format("Multipart object ETag: %s", this.eTag)); } - Optional validationError = validateResponse(response, partIndex); - if (validationError.isPresent()) { - log.debug(() -> "Response validation failed", validationError.get()); - asyncResponseTransformer.exceptionOccurred(validationError.get()); - handleError(validationError.get()); - return false; - } - if (totalContentLength == null && responseContentRange != null) { Optional parsedContentLength = MultipartDownloadUtils.parseContentRangeForTotalSize(responseContentRange); if (!parsedContentLength.isPresent()) { @@ -159,6 +154,15 @@ private boolean validatePart(GetObjectResponse response, int partIndex, this.totalParts = calculateTotalParts(totalContentLength, configuredPartSizeInBytes); log.debug(() -> String.format("Total content length: %d, Total parts: %d", totalContentLength, totalParts)); } + + Optional validationError = validateResponse(response, partIndex); + if (validationError.isPresent()) { + log.debug(() -> "Response validation failed", validationError.get()); + asyncResponseTransformer.exceptionOccurred(validationError.get()); + handleError(validationError.get()); + return false; + } + return true; } @@ -186,7 +190,7 @@ private Optional validateResponse(GetObjectResponse response if (contentRange == null) { return Optional.of(PresignedUrlDownloadHelper.missingContentRangeHeader()); } - + Long contentLength = response.contentLength(); if (contentLength == null || contentLength < 0) { return Optional.of(PresignedUrlDownloadHelper.invalidContentLength()); @@ -202,7 +206,7 @@ private Optional validateResponse(GetObjectResponse response String expectedRange = "bytes " + expectedStartByte + "-" + expectedEndByte + "/"; if (!contentRange.startsWith(expectedRange)) { return Optional.of(SdkClientException.create( - "Content-Range mismatch. Expected range starting with: " + expectedRange + + "Content-Range mismatch. Expected range starting with: " + expectedRange + ", but got: " + contentRange)); } @@ -215,19 +219,19 @@ private Optional validateResponse(GetObjectResponse response if (!contentLength.equals(expectedPartSize)) { return Optional.of(SdkClientException.create( String.format("Part content length validation failed for part %d. Expected: %d, but got: %d", - partIndex, expectedPartSize, contentLength))); + partIndex, expectedPartSize, contentLength))); } long actualStartByte = MultipartDownloadUtils.parseStartByteFromContentRange(contentRange); if (actualStartByte != expectedStartByte) { return Optional.of(SdkClientException.create( - "Content range offset mismatch for part " + partIndex + + "Content range offset mismatch for part " + partIndex + ". Expected start: " + expectedStartByte + ", but got: " + actualStartByte)); } - + return Optional.empty(); } - + private int calculateTotalParts(long contentLength, long partSize) { return (int) Math.ceil((double) contentLength / partSize); } @@ -246,7 +250,7 @@ private PresignedUrlDownloadRequest createRangedGetRequest(int partIndex) { } String rangeHeader = BYTES_RANGE_PREFIX + startByte + "-" + endByte; PresignedUrlDownloadRequest.Builder builder = presignedUrlDownloadRequest.toBuilder() - .range(rangeHeader); + .range(rangeHeader); if (partIndex > 0 && eTag != null) { builder.ifMatch(eTag); } @@ -254,18 +258,24 @@ private PresignedUrlDownloadRequest createRangedGetRequest(int partIndex) { } private void handleError(Throwable t) { + future.completeExceptionally(t); + if (resultFuture != null) { + resultFuture.completeExceptionally(t); + } synchronized (lock) { if (subscription != null) { subscription.cancel(); } } - onError(t); } @Override public void onError(Throwable t) { log.debug(() -> "Error in multipart download", t); future.completeExceptionally(t); + if (resultFuture != null) { + resultFuture.completeExceptionally(t); + } } @Override diff --git a/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/multipart/ParallelPresignedUrlMultipartDownloaderSubscriberTckTest.java b/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/multipart/ParallelPresignedUrlMultipartDownloaderSubscriberTckTest.java new file mode 100644 index 000000000000..bb05a0833a2e --- /dev/null +++ b/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/multipart/ParallelPresignedUrlMultipartDownloaderSubscriberTckTest.java @@ -0,0 +1,134 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.services.s3.internal.multipart; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.when; + +import java.net.MalformedURLException; +import java.nio.ByteBuffer; +import java.util.concurrent.CompletableFuture; +import org.mockito.Mockito; +import org.reactivestreams.Subscriber; +import org.reactivestreams.Subscription; +import org.reactivestreams.tck.SubscriberWhiteboxVerification; +import org.reactivestreams.tck.TestEnvironment; +import software.amazon.awssdk.core.async.AsyncResponseTransformer; +import software.amazon.awssdk.core.async.SdkPublisher; +import software.amazon.awssdk.services.s3.S3AsyncClient; +import software.amazon.awssdk.services.s3.model.GetObjectResponse; +import software.amazon.awssdk.services.s3.presignedurl.AsyncPresignedUrlExtension; +import software.amazon.awssdk.services.s3.presignedurl.model.PresignedUrlDownloadRequest; + +public class ParallelPresignedUrlMultipartDownloaderSubscriberTckTest + extends SubscriberWhiteboxVerification> { + + private final S3AsyncClient s3mock; + + public ParallelPresignedUrlMultipartDownloaderSubscriberTckTest() { + super(new TestEnvironment()); + this.s3mock = Mockito.mock(S3AsyncClient.class); + AsyncPresignedUrlExtension presignedUrlExtension = Mockito.mock(AsyncPresignedUrlExtension.class); + when(s3mock.presignedUrlExtension()).thenReturn(presignedUrlExtension); + + when(presignedUrlExtension.getObject(any(PresignedUrlDownloadRequest.class), any(AsyncResponseTransformer.class))) + .thenReturn(CompletableFuture.completedFuture( + GetObjectResponse.builder() + .contentRange("bytes 0-8388607/33554432") + .contentLength(8388608L) + .eTag("\"test-etag\"") + .build())); + } + + @Override + public Subscriber> createSubscriber( + WhiteboxSubscriberProbe> probe) { + + return new ParallelPresignedUrlMultipartDownloaderSubscriber( + s3mock, + createTestPresignedUrlRequest(), + 8 * 1024 * 1024L, + new CompletableFuture<>(), + 10 + ) { + @Override + public void onSubscribe(Subscription s) { + super.onSubscribe(s); + probe.registerOnSubscribe(new SubscriberPuppet() { + @Override + public void triggerRequest(long elements) { + s.request(elements); + } + + @Override + public void signalCancel() { + s.cancel(); + } + }); + } + + @Override + public void onNext(AsyncResponseTransformer item) { + super.onNext(item); + probe.registerOnNext(item); + } + + @Override + public void onError(Throwable t) { + super.onError(t); + probe.registerOnError(t); + } + + @Override + public void onComplete() { + super.onComplete(); + probe.registerOnComplete(); + } + }; + } + + @Override + public AsyncResponseTransformer createElement(int element) { + return new AsyncResponseTransformer() { + @Override + public CompletableFuture prepare() { + return new CompletableFuture<>(); + } + + @Override + public void onResponse(GetObjectResponse response) { + } + + @Override + public void onStream(SdkPublisher publisher) { + } + + @Override + public void exceptionOccurred(Throwable error) { + } + }; + } + + private PresignedUrlDownloadRequest createTestPresignedUrlRequest() { + try { + return PresignedUrlDownloadRequest.builder() + .presignedUrl(java.net.URI.create("https://test-bucket.s3.amazonaws.com/test-key").toURL()) + .build(); + } catch (MalformedURLException e) { + throw new RuntimeException(e); + } + } +} diff --git a/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/multipart/ParallelPresignedUrlMultipartDownloaderSubscriberTest.java b/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/multipart/ParallelPresignedUrlMultipartDownloaderSubscriberTest.java new file mode 100644 index 000000000000..0062029e1b83 --- /dev/null +++ b/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/multipart/ParallelPresignedUrlMultipartDownloaderSubscriberTest.java @@ -0,0 +1,250 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.services.s3.internal.multipart; + +import static com.github.tomakehurst.wiremock.client.WireMock.aResponse; +import static com.github.tomakehurst.wiremock.client.WireMock.get; +import static com.github.tomakehurst.wiremock.client.WireMock.getRequestedFor; +import static com.github.tomakehurst.wiremock.client.WireMock.matching; +import static com.github.tomakehurst.wiremock.client.WireMock.stubFor; +import static com.github.tomakehurst.wiremock.client.WireMock.urlEqualTo; +import static com.github.tomakehurst.wiremock.client.WireMock.verify; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import com.github.tomakehurst.wiremock.junit5.WireMockRuntimeInfo; +import com.github.tomakehurst.wiremock.junit5.WireMockTest; +import java.io.IOException; +import java.net.MalformedURLException; +import java.net.URI; +import java.net.URL; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Arrays; +import java.util.UUID; +import java.util.concurrent.CompletionException; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import software.amazon.awssdk.core.async.AsyncResponseTransformer; +import software.amazon.awssdk.core.exception.SdkClientException; +import software.amazon.awssdk.services.s3.S3AsyncClient; +import software.amazon.awssdk.services.s3.model.GetObjectResponse; +import software.amazon.awssdk.services.s3.multipart.MultipartConfiguration; +import software.amazon.awssdk.services.s3.presignedurl.model.PresignedUrlDownloadRequest; + +/** + * Unit tests for {@link ParallelPresignedUrlMultipartDownloaderSubscriber}. + * Tests parallel-specific behavior: single-part detection, concurrent requests, + * and error propagation with in-flight cancellation. + */ +@WireMockTest +class ParallelPresignedUrlMultipartDownloaderSubscriberTest { + + private static final String PRESIGNED_URL_PATH = "/parallel-test"; + private static final byte[] TEST_DATA = "ABCDEFGHIJKLMNOPQRSTUVWXYZ123456".getBytes(StandardCharsets.UTF_8); // 32 bytes + + private S3AsyncClient s3AsyncClient; + private URL presignedUrl; + private Path tempFile; + + @BeforeEach + void setup(WireMockRuntimeInfo wiremock) throws MalformedURLException { + MultipartConfiguration multipartConfig = MultipartConfiguration.builder() + .minimumPartSizeInBytes(16L) + .build(); + s3AsyncClient = S3AsyncClient.builder() + .endpointOverride(URI.create("http://localhost:" + wiremock.getHttpPort())) + .multipartEnabled(true) + .multipartConfiguration(multipartConfig) + .build(); + presignedUrl = new URL("http://localhost:" + wiremock.getHttpPort() + PRESIGNED_URL_PATH); + } + + @AfterEach + void cleanup() throws IOException { + if (tempFile != null && Files.exists(tempFile)) { + Files.delete(tempFile); + } + } + + @Test + void singlePartObject_shouldCompleteWithoutAdditionalRequests() throws IOException { + byte[] smallData = "0123456789".getBytes(StandardCharsets.UTF_8); + stubFor(get(urlEqualTo(PRESIGNED_URL_PATH)) + .withHeader("Range", matching("bytes=0-15")) + .willReturn(aResponse() + .withStatus(206) + .withHeader("Content-Length", String.valueOf(smallData.length)) + .withHeader("Content-Range", "bytes 0-9/10") + .withHeader("ETag", "\"single-part-etag\"") + .withBody(smallData))); + + tempFile = createTempFile(); + PresignedUrlDownloadRequest request = PresignedUrlDownloadRequest.builder() + .presignedUrl(presignedUrl) + .build(); + GetObjectResponse response = (GetObjectResponse) s3AsyncClient.presignedUrlExtension() + .getObject(request, AsyncResponseTransformer.toFile(tempFile)) + .join(); + + assertThat(response.eTag()).isEqualTo("\"single-part-etag\""); + assertThat(Files.readAllBytes(tempFile)).isEqualTo(smallData); + verify(1, getRequestedFor(urlEqualTo(PRESIGNED_URL_PATH))); + } + + @Test + void multiPartObject_shouldDownloadAllPartsConcurrently() throws IOException { + stubFor(get(urlEqualTo(PRESIGNED_URL_PATH)) + .withHeader("Range", matching("bytes=0-15")) + .willReturn(aResponse() + .withStatus(206) + .withHeader("Content-Length", "16") + .withHeader("Content-Range", "bytes 0-15/32") + .withHeader("ETag", "\"multi-etag\"") + .withBody(Arrays.copyOfRange(TEST_DATA, 0, 16)))); + + stubFor(get(urlEqualTo(PRESIGNED_URL_PATH)) + .withHeader("Range", matching("bytes=16-31")) + .willReturn(aResponse() + .withStatus(206) + .withHeader("Content-Length", "16") + .withHeader("Content-Range", "bytes 16-31/32") + .withHeader("ETag", "\"multi-etag\"") + .withBody(Arrays.copyOfRange(TEST_DATA, 16, 32)))); + + tempFile = createTempFile(); + PresignedUrlDownloadRequest request = PresignedUrlDownloadRequest.builder() + .presignedUrl(presignedUrl) + .build(); + s3AsyncClient.presignedUrlExtension() + .getObject(request, AsyncResponseTransformer.toFile(tempFile)) + .join(); + + assertThat(Files.readAllBytes(tempFile)).isEqualTo(TEST_DATA); + } + + @Test + void errorOnSecondPart_shouldCompleteExceptionallyAndNotSendMoreRequests() { + stubFor(get(urlEqualTo(PRESIGNED_URL_PATH)) + .withHeader("Range", matching("bytes=0-15")) + .willReturn(aResponse() + .withStatus(206) + .withHeader("Content-Length", "16") + .withHeader("Content-Range", "bytes 0-15/48") + .withHeader("ETag", "\"error-etag\"") + .withBody(Arrays.copyOfRange(TEST_DATA, 0, 16)))); + + stubFor(get(urlEqualTo(PRESIGNED_URL_PATH)) + .withHeader("Range", matching("bytes=16-31")) + .willReturn(aResponse() + .withStatus(500) + .withBody("InternalError" + + "Simulated failure"))); + + tempFile = createTempFileUnchecked(); + PresignedUrlDownloadRequest request = PresignedUrlDownloadRequest.builder() + .presignedUrl(presignedUrl) + .build(); + + assertThatThrownBy(() -> s3AsyncClient.presignedUrlExtension() + .getObject(request, AsyncResponseTransformer.toFile(tempFile)) + .join()) + .isInstanceOf(CompletionException.class); + } + + @Test + void missingContentRangeOnFirstPart_shouldFail() { + stubFor(get(urlEqualTo(PRESIGNED_URL_PATH)) + .willReturn(aResponse() + .withStatus(206) + .withHeader("Content-Length", "16") + .withHeader("ETag", "\"no-range-etag\"") + .withBody(Arrays.copyOfRange(TEST_DATA, 0, 16)))); + + tempFile = createTempFileUnchecked(); + PresignedUrlDownloadRequest request = PresignedUrlDownloadRequest.builder() + .presignedUrl(presignedUrl) + .build(); + + assertThatThrownBy(() -> s3AsyncClient.presignedUrlExtension() + .getObject(request, AsyncResponseTransformer.toFile(tempFile)) + .join()) + .hasRootCauseInstanceOf(SdkClientException.class) + .hasMessageContaining("No Content-Range header"); + } + + @Test + void contentRangeMismatchOnSecondPart_shouldFail() { + stubFor(get(urlEqualTo(PRESIGNED_URL_PATH)) + .withHeader("Range", matching("bytes=0-15")) + .willReturn(aResponse() + .withStatus(206) + .withHeader("Content-Length", "16") + .withHeader("Content-Range", "bytes 0-15/32") + .withHeader("ETag", "\"mismatch-etag\"") + .withBody(Arrays.copyOfRange(TEST_DATA, 0, 16)))); + + stubFor(get(urlEqualTo(PRESIGNED_URL_PATH)) + .withHeader("Range", matching("bytes=16-31")) + .willReturn(aResponse() + .withStatus(206) + .withHeader("Content-Length", "16") + .withHeader("Content-Range", "bytes 9999-10014/32") + .withHeader("ETag", "\"mismatch-etag\"") + .withBody(Arrays.copyOfRange(TEST_DATA, 16, 32)))); + + tempFile = createTempFileUnchecked(); + PresignedUrlDownloadRequest request = PresignedUrlDownloadRequest.builder() + .presignedUrl(presignedUrl) + .build(); + + assertThatThrownBy(() -> s3AsyncClient.presignedUrlExtension() + .getObject(request, AsyncResponseTransformer.toFile(tempFile)) + .join()) + .hasRootCauseInstanceOf(SdkClientException.class) + .hasMessageContaining("Content-Range mismatch"); + } + + @Test + void onNext_withNullTransformer_shouldThrowNPE() { + ParallelPresignedUrlMultipartDownloaderSubscriber subscriber = + new ParallelPresignedUrlMultipartDownloaderSubscriber( + s3AsyncClient, + PresignedUrlDownloadRequest.builder().presignedUrl(presignedUrl).build(), + 16L, + new java.util.concurrent.CompletableFuture<>(), + 10); + + assertThatThrownBy(() -> subscriber.onNext(null)) + .isInstanceOf(NullPointerException.class); + } + + private static Path createTempFile() throws IOException { + Path path = Files.createTempFile("parallel-test-" + UUID.randomUUID(), ".tmp"); + Files.deleteIfExists(path); + return path; + } + + private static Path createTempFileUnchecked() { + try { + return createTempFile(); + } catch (IOException e) { + throw new RuntimeException(e); + } + } +} diff --git a/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/multipart/PresignedUrlMultipartDownloaderSubscriberTckTest.java b/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/multipart/PresignedUrlMultipartDownloaderSubscriberTckTest.java index 1b28d0175395..3c176d7e635a 100644 --- a/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/multipart/PresignedUrlMultipartDownloaderSubscriberTckTest.java +++ b/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/multipart/PresignedUrlMultipartDownloaderSubscriberTckTest.java @@ -74,7 +74,8 @@ public PresignedUrlMultipartDownloaderSubscriberTckTest() { return new PresignedUrlMultipartDownloaderSubscriber( s3mock, createTestPresignedUrlRequest(), - 8 * 1024 * 1024L + 8 * 1024 * 1024L, + new CompletableFuture<>() ) { @Override public void onError(Throwable throwable) { diff --git a/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/multipart/PresignedUrlMultipartDownloaderSubscriberWiremockTest.java b/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/multipart/PresignedUrlMultipartDownloaderSubscriberWiremockTest.java index 3228ec2b8800..2ff0e4da46b7 100644 --- a/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/multipart/PresignedUrlMultipartDownloaderSubscriberWiremockTest.java +++ b/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/multipart/PresignedUrlMultipartDownloaderSubscriberWiremockTest.java @@ -35,12 +35,19 @@ import java.nio.file.Path; import java.util.Arrays; import java.util.UUID; +import java.util.concurrent.CompletableFuture; +import java.util.stream.Stream; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import software.amazon.awssdk.core.ResponseBytes; import software.amazon.awssdk.core.async.AsyncResponseTransformer; import software.amazon.awssdk.core.exception.SdkClientException; import software.amazon.awssdk.services.s3.S3AsyncClient; +import software.amazon.awssdk.services.s3.model.GetObjectResponse; import software.amazon.awssdk.services.s3.model.S3Exception; import software.amazon.awssdk.services.s3.multipart.MultipartConfiguration; import software.amazon.awssdk.services.s3.presignedurl.model.PresignedUrlDownloadRequest; @@ -70,156 +77,187 @@ public void setup(WireMockRuntimeInfo wiremock) throws MalformedURLException { presignedUrl = createPresignedUrl(); } - @Test - void presignedUrlDownload_withMultipartData_shouldReceiveCompleteBody() { + static Stream transformerTypes() { + return Stream.of( + Arguments.of("toBytes"), + Arguments.of("toFile") + ); + } + + private CompletableFuture executeDownload(PresignedUrlDownloadRequest request, String transformerType) + throws IOException { + if ("toFile".equals(transformerType)) { + tempFile = createUniqueTempFile(); + return s3AsyncClient.presignedUrlExtension().getObject(request, AsyncResponseTransformer.toFile(tempFile)); + } + return s3AsyncClient.presignedUrlExtension().getObject(request, AsyncResponseTransformer.toBytes()); + } + + @SuppressWarnings("unchecked") + private void assertSuccessfulDownload(String type, Object result) throws IOException { + if ("toBytes".equals(type)) { + assertArrayEquals(TEST_DATA, ((ResponseBytes) result).asByteArray()); + } else { + assertThat(tempFile.toFile()).exists(); + byte[] fileContent = Files.readAllBytes(tempFile); + assertArrayEquals(TEST_DATA, fileContent); + } + } + + @ParameterizedTest(name = "presignedUrlDownload_withMultipartData_shouldReceiveCompleteBody [{0}]") + @MethodSource("transformerTypes") + void presignedUrlDownload_withMultipartData_shouldReceiveCompleteBody(String transformerType) throws IOException { stubSuccessfulPresignedUrlResponse(); - byte[] result = s3AsyncClient.presignedUrlExtension() - .getObject(PresignedUrlDownloadRequest.builder() - .presignedUrl(presignedUrl) - .build(), - AsyncResponseTransformer.toBytes()) - .join() - .asByteArray(); - assertArrayEquals(TEST_DATA, result); + PresignedUrlDownloadRequest request = PresignedUrlDownloadRequest.builder() + .presignedUrl(presignedUrl) + .build(); + Object result = executeDownload(request, transformerType).join(); + assertSuccessfulDownload(transformerType, result); } - @Test - void presignedUrlDownload_withRangeHeader_shouldReceivePartialContent() { + @ParameterizedTest(name = "presignedUrlDownload_smallObjectSmallerThanPartSize_shouldSucceed [{0}]") + @MethodSource("transformerTypes") + @SuppressWarnings("unchecked") + void presignedUrlDownload_smallObjectSmallerThanPartSize_shouldSucceed(String transformerType) throws IOException { + byte[] smallData = "0123456789".getBytes(StandardCharsets.UTF_8); + stubFor(get(urlEqualTo(PRESIGNED_URL_PATH)) + .willReturn(aResponse() + .withStatus(206) + .withHeader("Content-Type", "application/octet-stream") + .withHeader("Content-Length", "10") + .withHeader("Content-Range", "bytes 0-9/10") + .withHeader("ETag", "\"small-etag\"") + .withBody(smallData))); + + PresignedUrlDownloadRequest request = PresignedUrlDownloadRequest.builder() + .presignedUrl(presignedUrl) + .build(); + Object result = executeDownload(request, transformerType).join(); + if ("toBytes".equals(transformerType)) { + assertArrayEquals(smallData, ((ResponseBytes) result).asByteArray()); + } else { + assertThat(tempFile.toFile()).exists(); + assertArrayEquals(smallData, Files.readAllBytes(tempFile)); + } + } + + @ParameterizedTest(name = "presignedUrlDownload_withRangeHeader_shouldReceivePartialContent [{0}]") + @MethodSource("transformerTypes") + @SuppressWarnings("unchecked") + void presignedUrlDownload_withRangeHeader_shouldReceivePartialContent(String transformerType) throws IOException { stubSuccessfulRangeResponse(); - byte[] result = s3AsyncClient.presignedUrlExtension() - .getObject(PresignedUrlDownloadRequest.builder() - .presignedUrl(presignedUrl) - .range("bytes=0-10") - .build(), - AsyncResponseTransformer.toBytes()) - .join() - .asByteArray(); + PresignedUrlDownloadRequest request = PresignedUrlDownloadRequest.builder() + .presignedUrl(presignedUrl) + .range("bytes=0-10") + .build(); + Object result = executeDownload(request, transformerType).join(); byte[] expectedPartial = Arrays.copyOfRange(TEST_DATA, 0, 11); - assertArrayEquals(expectedPartial, result); + if ("toBytes".equals(transformerType)) { + assertArrayEquals(expectedPartial, ((ResponseBytes) result).asByteArray()); + } else { + byte[] fileContent = Files.readAllBytes(tempFile); + assertArrayEquals(expectedPartial, fileContent); + } } - @Test - void presignedUrlDownload_whenRequestFails_shouldThrowException() { + @ParameterizedTest(name = "presignedUrlDownload_whenRequestFails_shouldThrowException [{0}]") + @MethodSource("transformerTypes") + void presignedUrlDownload_whenRequestFails_shouldThrowException(String transformerType) { stubFailedPresignedUrlResponse(); - assertThatThrownBy(() -> s3AsyncClient.presignedUrlExtension() - .getObject(PresignedUrlDownloadRequest.builder() - .presignedUrl(presignedUrl) - .build(), - AsyncResponseTransformer.toBytes()) - .join()) + PresignedUrlDownloadRequest request = PresignedUrlDownloadRequest.builder() + .presignedUrl(presignedUrl) + .build(); + assertThatThrownBy(() -> executeDownload(request, transformerType).join()) .hasRootCauseInstanceOf(S3Exception.class); } - @Test - void presignedUrlDownload_withFileTransformer_shouldWork() throws IOException { - stubSuccessfulPresignedUrlResponse(); - tempFile = createUniqueTempFile(); - s3AsyncClient.presignedUrlExtension() - .getObject(PresignedUrlDownloadRequest.builder() - .presignedUrl(presignedUrl) - .build(), - AsyncResponseTransformer.toFile(tempFile)) - .join(); - assertThat(tempFile.toFile()).exists(); - assertThat(tempFile.toFile().length()).isGreaterThan(0); - } - - @Test - void presignedUrlDownload_whenFirstRequestFails_shouldThrowException() { + @ParameterizedTest(name = "presignedUrlDownload_whenFirstRequestFails_shouldThrowException [{0}]") + @MethodSource("transformerTypes") + void presignedUrlDownload_whenFirstRequestFails_shouldThrowException(String transformerType) { stubInternalServerError(); - assertThatThrownBy(() -> s3AsyncClient.presignedUrlExtension() - .getObject(PresignedUrlDownloadRequest.builder() - .presignedUrl(presignedUrl) - .build(), - AsyncResponseTransformer.toBytes()) - .join()) + PresignedUrlDownloadRequest request = PresignedUrlDownloadRequest.builder() + .presignedUrl(presignedUrl) + .build(); + assertThatThrownBy(() -> executeDownload(request, transformerType).join()) .hasRootCauseInstanceOf(S3Exception.class); } - @Test - void presignedUrlDownload_whenSecondRequestFails_shouldThrowException() { + @ParameterizedTest(name = "presignedUrlDownload_whenSecondRequestFails_shouldThrowException [{0}]") + @MethodSource("transformerTypes") + void presignedUrlDownload_whenSecondRequestFails_shouldThrowException(String transformerType) { stubPartialFailureScenario(); - assertThatThrownBy(() -> s3AsyncClient.presignedUrlExtension() - .getObject(PresignedUrlDownloadRequest.builder() - .presignedUrl(presignedUrl) - .build(), - AsyncResponseTransformer.toBytes()) - .join()) + PresignedUrlDownloadRequest request = PresignedUrlDownloadRequest.builder() + .presignedUrl(presignedUrl) + .build(); + assertThatThrownBy(() -> executeDownload(request, transformerType).join()) .hasRootCauseInstanceOf(S3Exception.class); } - @Test - void presignedUrlDownload_whenIOErrorOccurs_shouldThrowException() { + @ParameterizedTest(name = "presignedUrlDownload_whenIOErrorOccurs_shouldThrowException [{0}]") + @MethodSource("transformerTypes") + void presignedUrlDownload_whenIOErrorOccurs_shouldThrowException(String transformerType) { stubConnectionReset(); - assertThatThrownBy(() -> s3AsyncClient.presignedUrlExtension() - .getObject(PresignedUrlDownloadRequest.builder() - .presignedUrl(presignedUrl) - .build(), - AsyncResponseTransformer.toBytes()) - .join()) - .hasCauseInstanceOf(IOException.class); + PresignedUrlDownloadRequest request = PresignedUrlDownloadRequest.builder() + .presignedUrl(presignedUrl) + .build(); + assertThatThrownBy(() -> executeDownload(request, transformerType).join()) + .hasCauseInstanceOf(SdkClientException.class); } - - @Test - void onNext_withNullTransformer_shouldThrowException() { - PresignedUrlMultipartDownloaderSubscriber subscriber = createTestSubscriber(); - - assertThatThrownBy(() -> subscriber.onNext(null)) - .isInstanceOf(NullPointerException.class) - .hasMessageContaining("onNext must not be called with null asyncResponseTransformer"); - } - - @Test - void presignedUrlDownload_withMissingContentRange_shouldFailRequest() { + @ParameterizedTest(name = "presignedUrlDownload_withMissingContentRange_shouldFailRequest [{0}]") + @MethodSource("transformerTypes") + void presignedUrlDownload_withMissingContentRange_shouldFailRequest(String transformerType) { stubResponseWithMissingContentRange(); - assertThatThrownBy(() -> s3AsyncClient.presignedUrlExtension() - .getObject(PresignedUrlDownloadRequest.builder() - .presignedUrl(presignedUrl) - .build(), - AsyncResponseTransformer.toBytes()) - .join()) + PresignedUrlDownloadRequest request = PresignedUrlDownloadRequest.builder() + .presignedUrl(presignedUrl) + .build(); + assertThatThrownBy(() -> executeDownload(request, transformerType).join()) .hasRootCauseInstanceOf(SdkClientException.class) .hasMessageContaining("No Content-Range header in response"); } - @Test - void presignedUrlDownload_withInvalidContentLength_shouldFailRequest() { + @ParameterizedTest(name = "presignedUrlDownload_withInvalidContentLength_shouldFailRequest [{0}]") + @MethodSource("transformerTypes") + void presignedUrlDownload_withInvalidContentLength_shouldFailRequest(String transformerType) { stubResponseWithInvalidContentLength(); - assertThatThrownBy(() -> s3AsyncClient.presignedUrlExtension() - .getObject(PresignedUrlDownloadRequest.builder() - .presignedUrl(presignedUrl) - .build(), - AsyncResponseTransformer.toBytes()) - .join()) + PresignedUrlDownloadRequest request = PresignedUrlDownloadRequest.builder() + .presignedUrl(presignedUrl) + .build(); + assertThatThrownBy(() -> executeDownload(request, transformerType).join()) .hasRootCauseInstanceOf(SdkClientException.class) .hasMessageContaining("Invalid or missing Content-Length in response"); } - @Test - void presignedUrlDownload_withContentRangeMismatch_shouldFailRequest() { + @ParameterizedTest(name = "presignedUrlDownload_withContentRangeMismatch_shouldFailRequest [{0}]") + @MethodSource("transformerTypes") + void presignedUrlDownload_withContentRangeMismatch_shouldFailRequest(String transformerType) { stubResponseWithContentRangeMismatch(); - assertThatThrownBy(() -> s3AsyncClient.presignedUrlExtension() - .getObject(PresignedUrlDownloadRequest.builder() - .presignedUrl(presignedUrl) - .build(), - AsyncResponseTransformer.toBytes()) - .join()) + PresignedUrlDownloadRequest request = PresignedUrlDownloadRequest.builder() + .presignedUrl(presignedUrl) + .build(); + assertThatThrownBy(() -> executeDownload(request, transformerType).join()) .hasRootCauseInstanceOf(SdkClientException.class) .hasMessageContaining("Content-Range mismatch"); } - @Test - void presignedUrlDownload_withContentLengthMismatch_shouldFailRequest() { + @ParameterizedTest(name = "presignedUrlDownload_withContentLengthMismatch_shouldFailRequest [{0}]") + @MethodSource("transformerTypes") + void presignedUrlDownload_withContentLengthMismatch_shouldFailRequest(String transformerType) { stubResponseWithContentLengthMismatch(); - assertThatThrownBy(() -> s3AsyncClient.presignedUrlExtension() - .getObject(PresignedUrlDownloadRequest.builder() - .presignedUrl(presignedUrl) - .build(), - AsyncResponseTransformer.toBytes()) - .join()) - .hasRootCauseInstanceOf(SdkClientException.class) - .hasMessageContaining("Part content length validation failed"); + PresignedUrlDownloadRequest request = PresignedUrlDownloadRequest.builder() + .presignedUrl(presignedUrl) + .build(); + assertThatThrownBy(() -> executeDownload(request, transformerType).join()) + .hasRootCauseInstanceOf(SdkClientException.class); + } + + @Test + void onNext_withNullTransformer_shouldThrowException() { + PresignedUrlMultipartDownloaderSubscriber subscriber = createTestSubscriber(); + + assertThatThrownBy(() -> subscriber.onNext(null)) + .isInstanceOf(NullPointerException.class) + .hasMessageContaining("onNext must not be called with null asyncResponseTransformer"); } @AfterEach @@ -250,7 +288,7 @@ private void stubSuccessfulPresignedUrlResponse() { .withHeader("Content-Range", "bytes 0-15/32") .withHeader("ETag", "\"test-etag\"") .withBody(Arrays.copyOfRange(TEST_DATA, 0, 16)))); - + // Stub for second part (bytes 16-31) stubFor(get(urlEqualTo(PRESIGNED_URL_PATH)) .withHeader("Range", matching("bytes=16-31")) @@ -324,7 +362,8 @@ private PresignedUrlMultipartDownloaderSubscriber createTestSubscriber() { return new PresignedUrlMultipartDownloaderSubscriber( s3AsyncClient, PresignedUrlDownloadRequest.builder().presignedUrl(presignedUrl).build(), - 1024L); + 1024L, + new CompletableFuture<>()); } private void stubResponseWithMissingContentRange() { @@ -368,4 +407,4 @@ private void stubResponseWithContentLengthMismatch() { .withHeader("ETag", "\"test-etag\"") .withBody(Arrays.copyOfRange(TEST_DATA, 0, 8)))); } -} \ No newline at end of file +} From b508bf060e66edf1b6e7640f39dd05312117e104 Mon Sep 17 00:00:00 2001 From: jencymaryjoseph <35571282+jencymaryjoseph@users.noreply.github.com> Date: Tue, 5 May 2026 14:13:01 -0700 Subject: [PATCH 14/30] Fix CodeBuild failure: update codegen file for EndpointResolverInterceptorSpec null check --- .../rules/endpoint-resolve-interceptor-with-stringarray.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/codegen/src/test/resources/software/amazon/awssdk/codegen/poet/rules/endpoint-resolve-interceptor-with-stringarray.java b/codegen/src/test/resources/software/amazon/awssdk/codegen/poet/rules/endpoint-resolve-interceptor-with-stringarray.java index 0f5033376ffc..fc8471b3bb4b 100644 --- a/codegen/src/test/resources/software/amazon/awssdk/codegen/poet/rules/endpoint-resolve-interceptor-with-stringarray.java +++ b/codegen/src/test/resources/software/amazon/awssdk/codegen/poet/rules/endpoint-resolve-interceptor-with-stringarray.java @@ -84,7 +84,7 @@ public SdkRequest modifyRequest(Context.ModifyRequest context, ExecutionAttribut @Override public SdkHttpRequest modifyHttpRequest(Context.ModifyHttpRequest context, ExecutionAttributes executionAttributes) { Endpoint resolvedEndpoint = executionAttributes.getAttribute(SdkInternalExecutionAttribute.RESOLVED_ENDPOINT); - if (resolvedEndpoint.headers().isEmpty()) { + if (resolvedEndpoint == null || CollectionUtils.isNullOrEmpty(resolvedEndpoint.headers())) { return context.httpRequest(); } SdkHttpRequest.Builder httpRequestBuilder = context.httpRequest().toBuilder(); From 10e45b13a5f999bc6c2d4c6b89aa7aec32c861cd Mon Sep 17 00:00:00 2001 From: jencymaryjoseph <35571282+jencymaryjoseph@users.noreply.github.com> Date: Fri, 8 May 2026 14:35:33 -0700 Subject: [PATCH 15/30] Address PR Comments --- ...signedUrlMultipartDownloaderSubscriber.java | 18 +++++++++++------- ...signedUrlMultipartDownloaderSubscriber.java | 13 ++++++++++++- 2 files changed, 23 insertions(+), 8 deletions(-) diff --git a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/ParallelPresignedUrlMultipartDownloaderSubscriber.java b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/ParallelPresignedUrlMultipartDownloaderSubscriber.java index 9c8f77ada905..8a7c898b3619 100644 --- a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/ParallelPresignedUrlMultipartDownloaderSubscriber.java +++ b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/ParallelPresignedUrlMultipartDownloaderSubscriber.java @@ -54,6 +54,11 @@ public class ParallelPresignedUrlMultipartDownloaderSubscriber private final S3AsyncClient s3AsyncClient; private final PresignedUrlDownloadRequest presignedUrlDownloadRequest; private final long configuredPartSizeInBytes; + /** + * The future returned by the SplitResult, representing the overall download completion. + * Completed exceptionally on error (before cancel) to prevent ByteArraySplittingTransformer + * from assembling invalid data. + */ private final CompletableFuture resultFuture; private final int maxInFlightParts; @@ -140,7 +145,6 @@ private void sendFirstRequest(AsyncResponseTransformer String.format("Total content length: %d, Total parts: %d", totalContentLength, totalParts)); if (totalParts <= 1) { + resultFuture.complete(firstResponse); synchronized (subscriptionLock) { subscription.cancel(); } - resultFuture.complete(firstResponse); return; } @@ -226,11 +230,11 @@ private void sendPartRequest(AsyncResponseTransformer= totalParts) { + if (totalComplete == totalParts) { + resultFuture.complete(firstResponse); synchronized (subscriptionLock) { subscription.cancel(); } - resultFuture.complete(firstResponse); } else { processPendingTransformers(); synchronized (subscriptionLock) { @@ -247,10 +251,10 @@ private void processPendingTransformers() { } try { while (!pendingTransformers.isEmpty() && inFlightRequestsNum.get() < maxInFlightParts) { - Pair> pair = + Pair> pendingPart = pendingTransformers.poll(); - if (pair != null && pair.left() < totalParts) { - sendPartRequest(pair.right(), pair.left()); + if (pendingPart != null && pendingPart.left() < totalParts) { + sendPartRequest(pendingPart.right(), pendingPart.left()); } } } finally { diff --git a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/PresignedUrlMultipartDownloaderSubscriber.java b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/PresignedUrlMultipartDownloaderSubscriber.java index 7c042ef5aefa..d76f0a8e8e6b 100644 --- a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/PresignedUrlMultipartDownloaderSubscriber.java +++ b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/PresignedUrlMultipartDownloaderSubscriber.java @@ -53,7 +53,19 @@ public class PresignedUrlMultipartDownloaderSubscriber private final S3AsyncClient s3AsyncClient; private final PresignedUrlDownloadRequest presignedUrlDownloadRequest; private final Long configuredPartSizeInBytes; + + /** + * Internal lifecycle future for this subscriber. Completed when all parts are downloaded + * or when an error occurs. + */ private final CompletableFuture future; + + /** + * The split transformer's completion future (from {@code SplitResult.resultFuture()}). + * Completing this signals the download result to the caller. Must be completed exceptionally + * before cancelling the subscription to prevent ByteArraySplittingTransformer from assembling + * invalid data. + */ private final CompletableFuture resultFuture; private final Object lock = new Object(); private final AtomicInteger completedParts; @@ -118,7 +130,6 @@ private void makeRangeRequest(int partIndex, .getObject(partRequest, asyncResponseTransformer) .whenComplete((response, error) -> { if (error != null) { - log.debug(() -> "Error encountered during part request for part " + partIndex); handleError(error); return; } From 7e1c3a06d53b546c5b510cf0ad0389d746d45c25 Mon Sep 17 00:00:00 2001 From: jencymaryjoseph <35571282+jencymaryjoseph@users.noreply.github.com> Date: Wed, 13 May 2026 09:41:52 -0700 Subject: [PATCH 16/30] Fix check-then-act race condition in parallel subscriber (#6956) * Fix check-then-act race condition in parallel subscriber * address comments --- ...ignedUrlMultipartDownloaderSubscriber.java | 34 ++++++++++----- ...dUrlMultipartDownloaderSubscriberTest.java | 42 +++++++++++++++++++ 2 files changed, 65 insertions(+), 11 deletions(-) diff --git a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/ParallelPresignedUrlMultipartDownloaderSubscriber.java b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/ParallelPresignedUrlMultipartDownloaderSubscriber.java index 8a7c898b3619..bf1e9ddaf6ab 100644 --- a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/ParallelPresignedUrlMultipartDownloaderSubscriber.java +++ b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/ParallelPresignedUrlMultipartDownloaderSubscriber.java @@ -21,6 +21,7 @@ import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.Semaphore; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import org.reactivestreams.Subscriber; @@ -64,7 +65,7 @@ public class ParallelPresignedUrlMultipartDownloaderSubscriber private final AtomicInteger partNumber = new AtomicInteger(0); private final AtomicInteger completedParts = new AtomicInteger(0); - private final AtomicInteger inFlightRequestsNum = new AtomicInteger(0); + private final Semaphore inFlightPermits; private final AtomicBoolean isCompletedExceptionally = new AtomicBoolean(false); private final AtomicBoolean processingPending = new AtomicBoolean(false); private final Map> inFlightRequests = new ConcurrentHashMap<>(); @@ -90,6 +91,7 @@ public ParallelPresignedUrlMultipartDownloaderSubscriber( this.configuredPartSizeInBytes = configuredPartSizeInBytes; this.resultFuture = resultFuture; this.maxInFlightParts = maxInFlightParts; + this.inFlightPermits = new Semaphore(maxInFlightParts); } @Override @@ -128,21 +130,25 @@ private void sendFirstRequest(AsyncResponseTransformer "Sending first range request with range=" + partRequest.range()); + if (!inFlightPermits.tryAcquire()) { + throw new IllegalStateException("Failed to acquire permit for first request"); + } + CompletableFuture response = s3AsyncClient.presignedUrlExtension().getObject(partRequest, transformer); inFlightRequests.put(0, response); - inFlightRequestsNum.incrementAndGet(); CompletableFutureUtils.forwardExceptionTo(resultFuture, response); response.whenComplete((res, error) -> { + inFlightRequests.remove(0); + inFlightPermits.release(); + if (error != null || isCompletedExceptionally.get()) { handlePartError(error, 0); return; } - inFlightRequests.remove(0); - inFlightRequestsNum.decrementAndGet(); completedParts.incrementAndGet(); this.eTag = res.eTag(); @@ -188,7 +194,7 @@ private void processRequest(AsyncResponseTransformer= maxInFlightParts) { + if (!inFlightPermits.tryAcquire()) { pendingTransformers.offer(Pair.of(currentPart, transformer)); return; } @@ -200,6 +206,7 @@ private void processRequest(AsyncResponseTransformer transformer, int partIndex) { if (isCompletedExceptionally.get()) { + inFlightPermits.release(); return; } @@ -210,10 +217,12 @@ private void sendPartRequest(AsyncResponseTransformer { + inFlightRequests.remove(partIndex); + inFlightPermits.release(); + if (error != null || isCompletedExceptionally.get()) { handlePartError(error, partIndex); return; @@ -226,8 +235,6 @@ private void sendPartRequest(AsyncResponseTransformer "Completed part: " + partIndex); - inFlightRequests.remove(partIndex); - inFlightRequestsNum.decrementAndGet(); int totalComplete = completedParts.incrementAndGet(); if (totalComplete == totalParts) { @@ -245,22 +252,27 @@ private void sendPartRequest(AsyncResponseTransformer> pendingPart = pendingTransformers.poll(); if (pendingPart != null && pendingPart.left() < totalParts) { sendPartRequest(pendingPart.right(), pendingPart.left()); + } else { + inFlightPermits.release(); } } } finally { processingPending.set(false); } - } while (!pendingTransformers.isEmpty() && inFlightRequestsNum.get() < maxInFlightParts); + } while (!pendingTransformers.isEmpty() && inFlightPermits.availablePermits() > 0); } private Optional validatePartResponse(GetObjectResponse response, int partIndex) { @@ -336,4 +348,4 @@ public void onError(Throwable t) { public void onComplete() { // Completion is handled by resultFuture } -} +} \ No newline at end of file diff --git a/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/multipart/ParallelPresignedUrlMultipartDownloaderSubscriberTest.java b/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/multipart/ParallelPresignedUrlMultipartDownloaderSubscriberTest.java index 0062029e1b83..f24b2b3a2d1b 100644 --- a/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/multipart/ParallelPresignedUrlMultipartDownloaderSubscriberTest.java +++ b/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/multipart/ParallelPresignedUrlMultipartDownloaderSubscriberTest.java @@ -16,6 +16,7 @@ package software.amazon.awssdk.services.s3.internal.multipart; import static com.github.tomakehurst.wiremock.client.WireMock.aResponse; +import static com.github.tomakehurst.wiremock.client.WireMock.findAll; import static com.github.tomakehurst.wiremock.client.WireMock.get; import static com.github.tomakehurst.wiremock.client.WireMock.getRequestedFor; import static com.github.tomakehurst.wiremock.client.WireMock.matching; @@ -27,6 +28,7 @@ import com.github.tomakehurst.wiremock.junit5.WireMockRuntimeInfo; import com.github.tomakehurst.wiremock.junit5.WireMockTest; +import com.github.tomakehurst.wiremock.verification.LoggedRequest; import java.io.IOException; import java.net.MalformedURLException; import java.net.URI; @@ -35,6 +37,7 @@ import java.nio.file.Files; import java.nio.file.Path; import java.util.Arrays; +import java.util.List; import java.util.UUID; import java.util.concurrent.CompletionException; import org.junit.jupiter.api.AfterEach; @@ -234,6 +237,45 @@ void onNext_withNullTransformer_shouldThrowNPE() { .isInstanceOf(NullPointerException.class); } + @Test + void multiPartDownload_manyParts_shouldCompleteSuccessfully() throws Exception { + // 13 parts to exceed maxInFlightParts (10) + byte[] data = new byte[208]; // 13 × 16 bytes + Arrays.fill(data, (byte) 'X'); + + stubFor(get(urlEqualTo(PRESIGNED_URL_PATH)) + .withHeader("Range", matching("bytes=0-15")) + .willReturn(aResponse().withStatus(206) + .withHeader("Content-Length", "16") + .withHeader("Content-Range", "bytes 0-15/208") + .withHeader("ETag", "\"etag\"") + .withBody(Arrays.copyOfRange(data, 0, 16)))); + + for (int i = 1; i < 13; i++) { + int start = i * 16; + int end = start + 15; + stubFor(get(urlEqualTo(PRESIGNED_URL_PATH)) + .withHeader("Range", matching("bytes=" + start + "-" + end)) + .willReturn(aResponse().withStatus(206) + .withHeader("Content-Length", "16") + .withHeader("Content-Range", "bytes " + start + "-" + end + "/208") + .withHeader("ETag", "\"etag\"") + .withBody(Arrays.copyOfRange(data, start, end + 1)))); + } + + tempFile = createTempFileUnchecked(); + PresignedUrlDownloadRequest request = PresignedUrlDownloadRequest.builder() + .presignedUrl(presignedUrl) + .build(); + + s3AsyncClient.presignedUrlExtension() + .getObject(request, AsyncResponseTransformer.toFile(tempFile)) + .join(); + + assertThat(Files.readAllBytes(tempFile)).isEqualTo(data); + verify(13, getRequestedFor(urlEqualTo(PRESIGNED_URL_PATH))); + } + private static Path createTempFile() throws IOException { Path path = Files.createTempFile("parallel-test-" + UUID.randomUUID(), ".tmp"); Files.deleteIfExists(path); From 4b75eefe6d39618a46d5304d8403f56cfd3db2ce Mon Sep 17 00:00:00 2001 From: jencymaryjoseph <35571282+jencymaryjoseph@users.noreply.github.com> Date: Mon, 18 May 2026 10:16:27 -0700 Subject: [PATCH 17/30] Add presigned URL download support to S3 Transfer Manager (#6951) * Add presigned URL download support to S3 Transfer Manager * Add presigned URL download methods to DelegatingS3TransferManager * Address PR Comments --- .../transfer/s3/S3IntegrationTestBase.java | 20 ++ ...erPresignedUrlDownloadIntegrationTest.java | 141 +++++++++++++ .../awssdk/transfer/s3/S3TransferManager.java | 103 ++++++++++ .../internal/DelegatingS3TransferManager.java | 12 ++ .../s3/internal/GenericS3TransferManager.java | 76 +++++++ ...ansferManagerPresignedUrlDownloadTest.java | 192 ++++++++++++++++++ 6 files changed, 544 insertions(+) create mode 100644 services-custom/s3-transfer-manager/src/it/java/software/amazon/awssdk/transfer/s3/S3TransferManagerPresignedUrlDownloadIntegrationTest.java create mode 100644 services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/S3TransferManagerPresignedUrlDownloadTest.java diff --git a/services-custom/s3-transfer-manager/src/it/java/software/amazon/awssdk/transfer/s3/S3IntegrationTestBase.java b/services-custom/s3-transfer-manager/src/it/java/software/amazon/awssdk/transfer/s3/S3IntegrationTestBase.java index d976aa6342cf..2c66d951a715 100644 --- a/services-custom/s3-transfer-manager/src/it/java/software/amazon/awssdk/transfer/s3/S3IntegrationTestBase.java +++ b/services-custom/s3-transfer-manager/src/it/java/software/amazon/awssdk/transfer/s3/S3IntegrationTestBase.java @@ -55,11 +55,13 @@ public class S3IntegrationTestBase extends AwsTestBase { protected static S3Client s3; protected static S3AsyncClient s3Async; + protected static S3AsyncClient s3NonMultipartAsync; protected static S3AsyncClient s3CrtAsync; protected static S3TransferManager tmCrt; protected static S3TransferManager tmJava; + protected static S3TransferManager tmNonMultipartJava; /** * Loads the AWS account info for the integration tests and creates an S3 @@ -71,6 +73,11 @@ public static void setUpForAllIntegTests() throws Exception { System.setProperty("aws.crt.debugnative", "true"); s3 = s3ClientBuilder().build(); s3Async = s3AsyncClientBuilder().build(); + s3NonMultipartAsync = S3AsyncClient.builder() + .credentialsProvider(CREDENTIALS_PROVIDER_CHAIN) + .region(DEFAULT_REGION) + .build(); + s3CrtAsync = S3CrtAsyncClient.builder() .credentialsProvider(CREDENTIALS_PROVIDER_CHAIN) .region(DEFAULT_REGION) @@ -82,6 +89,9 @@ public static void setUpForAllIntegTests() throws Exception { tmJava = S3TransferManager.builder() .s3Client(s3Async) .build(); + tmNonMultipartJava = S3TransferManager.builder() + .s3Client(s3NonMultipartAsync) + .build(); } @@ -89,8 +99,11 @@ public static void setUpForAllIntegTests() throws Exception { public static void cleanUpForAllIntegTests() { s3.close(); s3Async.close(); + s3NonMultipartAsync.close(); s3CrtAsync.close(); tmCrt.close(); + tmJava.close(); + tmNonMultipartJava.close(); CrtResource.waitForNoResources(); } @@ -182,4 +195,11 @@ static Stream transferManagers() { Arguments.of(tmJava)); } + static Stream presignedUrlTransferManagers() { + return Stream.of( + // TODO: uncomment when CRT for presigned URL is supported + // Arguments.of(tmCrt), + Arguments.of(tmNonMultipartJava), + Arguments.of(tmJava)); + } } diff --git a/services-custom/s3-transfer-manager/src/it/java/software/amazon/awssdk/transfer/s3/S3TransferManagerPresignedUrlDownloadIntegrationTest.java b/services-custom/s3-transfer-manager/src/it/java/software/amazon/awssdk/transfer/s3/S3TransferManagerPresignedUrlDownloadIntegrationTest.java new file mode 100644 index 000000000000..0fa70b782439 --- /dev/null +++ b/services-custom/s3-transfer-manager/src/it/java/software/amazon/awssdk/transfer/s3/S3TransferManagerPresignedUrlDownloadIntegrationTest.java @@ -0,0 +1,141 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.transfer.s3; + +import static org.assertj.core.api.Assertions.assertThat; +import static software.amazon.awssdk.testutils.service.S3BucketUtils.temporaryBucketName; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Duration; +import java.util.stream.Stream; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import software.amazon.awssdk.core.ResponseBytes; +import software.amazon.awssdk.core.async.AsyncResponseTransformer; +import software.amazon.awssdk.services.s3.model.GetObjectRequest; +import software.amazon.awssdk.services.s3.model.GetObjectResponse; +import software.amazon.awssdk.services.s3.model.PutObjectRequest; +import software.amazon.awssdk.services.s3.presigner.S3Presigner; +import software.amazon.awssdk.services.s3.presigner.model.GetObjectPresignRequest; +import software.amazon.awssdk.services.s3.presigner.model.PresignedGetObjectRequest; +import software.amazon.awssdk.services.s3.presignedurl.model.PresignedUrlDownloadRequest; +import software.amazon.awssdk.testutils.RandomTempFile; +import software.amazon.awssdk.transfer.s3.model.CompletedDownload; +import software.amazon.awssdk.transfer.s3.model.CompletedFileDownload; +import software.amazon.awssdk.transfer.s3.model.FileDownload; +import software.amazon.awssdk.transfer.s3.model.PresignedDownloadFileRequest; +import software.amazon.awssdk.transfer.s3.model.PresignedDownloadRequest; +import software.amazon.awssdk.transfer.s3.progress.LoggingTransferListener; +import software.amazon.awssdk.utils.Md5Utils; + +public class S3TransferManagerPresignedUrlDownloadIntegrationTest extends S3IntegrationTestBase { + private static final String BUCKET = temporaryBucketName(S3TransferManagerPresignedUrlDownloadIntegrationTest.class); + private static final String SMALL_KEY = "small-key"; + private static final String LARGE_KEY = "large-key"; + private static final int SMALL_OBJ_SIZE = 5 * 1024 * 1024; + private static final int LARGE_OBJ_SIZE = 16 * 1024 * 1024; + + private static File smallFile; + private static File largeFile; + private static S3Presigner presigner; + + @BeforeAll + public static void setup() throws IOException { + createBucket(BUCKET); + smallFile = new RandomTempFile(SMALL_OBJ_SIZE); + largeFile = new RandomTempFile(LARGE_OBJ_SIZE); + s3.putObject(PutObjectRequest.builder().bucket(BUCKET).key(SMALL_KEY).build(), smallFile.toPath()); + s3.putObject(PutObjectRequest.builder().bucket(BUCKET).key(LARGE_KEY).build(), largeFile.toPath()); + presigner = S3Presigner.builder() + .region(DEFAULT_REGION) + .credentialsProvider(CREDENTIALS_PROVIDER_CHAIN) + .build(); + } + + @AfterAll + public static void cleanup() { + if (presigner != null) { + presigner.close(); + } + deleteBucketAndAllContents(BUCKET); + } + + static Stream testCases() { + return presignedUrlTransferManagers().flatMap(tmArg -> Stream.of( + Arguments.of(tmArg.get()[0], SMALL_KEY, smallFile, SMALL_OBJ_SIZE), + Arguments.of(tmArg.get()[0], LARGE_KEY, largeFile, LARGE_OBJ_SIZE) + )); + } + + @ParameterizedTest(name = "downloadFileWithPresignedUrl_{1}") + @MethodSource("testCases") + void downloadFileWithPresignedUrl_shouldDownloadCorrectly(S3TransferManager tm, String key, + File sourceFile, int objSize) throws Exception { + Path downloadPath = RandomTempFile.randomUncreatedFile().toPath(); + FileDownload download = tm.downloadFileWithPresignedUrl(createFileDownloadRequest(key, downloadPath)); + CompletedFileDownload completed = download.completionFuture().join(); + + assertThat(Files.exists(downloadPath)).isTrue(); + assertThat(Md5Utils.md5AsBase64(downloadPath.toFile())).isEqualTo(Md5Utils.md5AsBase64(sourceFile)); + assertThat(completed.response().responseMetadata().requestId()).isNotNull(); + } + + @ParameterizedTest(name = "downloadWithPresignedUrl_toBytes_{1}") + @MethodSource("testCases") + void downloadWithPresignedUrl_toBytes_shouldReturnCorrectData(S3TransferManager tm, String key, + File sourceFile, int objSize) { + CompletedDownload> completed = + tm.downloadWithPresignedUrl(createBytesDownloadRequest(key)).completionFuture().join(); + + assertThat(completed.result().asByteArray()).hasSize(objSize); + } + + private static PresignedDownloadFileRequest createFileDownloadRequest(String key, Path destination) { + return PresignedDownloadFileRequest.builder() + .presignedUrlDownloadRequest(PresignedUrlDownloadRequest.builder() + .presignedUrl(createPresignedRequest(key).url()) + .build()) + .destination(destination) + .addTransferListener(LoggingTransferListener.create()) + .build(); + } + + private static PresignedDownloadRequest> createBytesDownloadRequest(String key) { + return PresignedDownloadRequest.builder() + .presignedUrlDownloadRequest(PresignedUrlDownloadRequest.builder() + .presignedUrl(createPresignedRequest(key).url()) + .build()) + .responseTransformer(AsyncResponseTransformer.toBytes()) + .addTransferListener(LoggingTransferListener.create()) + .build(); + } + + private static PresignedGetObjectRequest createPresignedRequest(String key) { + return presigner.presignGetObject(GetObjectPresignRequest.builder() + .signatureDuration(Duration.ofMinutes(10)) + .getObjectRequest(GetObjectRequest.builder() + .bucket(BUCKET) + .key(key) + .build()) + .build()); + } +} diff --git a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/S3TransferManager.java b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/S3TransferManager.java index b5c0e823a91a..42bde7ecdd23 100644 --- a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/S3TransferManager.java +++ b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/S3TransferManager.java @@ -41,6 +41,8 @@ import software.amazon.awssdk.transfer.s3.model.DownloadRequest; import software.amazon.awssdk.transfer.s3.model.FileDownload; import software.amazon.awssdk.transfer.s3.model.FileUpload; +import software.amazon.awssdk.transfer.s3.model.PresignedDownloadFileRequest; +import software.amazon.awssdk.transfer.s3.model.PresignedDownloadRequest; import software.amazon.awssdk.transfer.s3.model.ResumableFileDownload; import software.amazon.awssdk.transfer.s3.model.ResumableFileUpload; import software.amazon.awssdk.transfer.s3.model.Upload; @@ -696,6 +698,107 @@ default Copy copy(Consumer copyRequestBuilder) { return copy(CopyRequest.builder().applyMutation(copyRequestBuilder).build()); } + /** + * Downloads an object using a pre-signed URL to a local file. For non-file-based downloads, you may use + * {@link #downloadWithPresignedUrl(PresignedDownloadRequest)} instead. + *

+ * This method supports multipart downloads when using a CRT-based or multipart-enabled S3 client, + * providing enhanced throughput and reliability for large objects. Progress can be monitored + * through {@link TransferListener}s attached to the request. + *

+ * The SDK will create a new file if the provided destination doesn't exist. If the file already exists, + * it will be replaced. In the event of an error, the SDK will NOT attempt to delete + * the file, leaving it as-is. + *

+ * Note: The result of the operation doesn't support pause and resume functionality. + *

+ *

+ * Usage Example: + * {@snippet : + * S3TransferManager transferManager = S3TransferManager.create(); + * + * // Create presigned URL (typically done by another service) + * PresignedUrlDownloadRequest presignedRequest = PresignedUrlDownloadRequest.builder() + * .presignedUrl(presignedUrl) + * .build(); + * + * PresignedDownloadFileRequest request = PresignedDownloadFileRequest.builder() + * .presignedUrlDownloadRequest(presignedRequest) + * .destination(Paths.get("downloaded-file.txt")) + * .addTransferListener( + * LoggingTransferListener.create()) + * .build(); + * + * FileDownload download = transferManager.downloadFileWithPresignedUrl(request); + * download.completionFuture().join(); + * } + * + * @param presignedDownloadFileRequest the presigned download file request + * @return A {@link FileDownload} that can be used to track the ongoing transfer + * @see #downloadFileWithPresignedUrl(Consumer) + * @see #downloadWithPresignedUrl(PresignedDownloadRequest) + */ + default FileDownload downloadFileWithPresignedUrl(PresignedDownloadFileRequest presignedDownloadFileRequest) { + throw new UnsupportedOperationException(); + } + + /** + * This is a convenience method that creates an instance of the {@link PresignedDownloadFileRequest} builder, + * avoiding the need to create one manually via {@link PresignedDownloadFileRequest#builder()}. + *

+ * Note: The result of the operation doesn't support pause and resume functionality. + *

+ * + * @see #downloadFileWithPresignedUrl(PresignedDownloadFileRequest) + */ + default FileDownload downloadFileWithPresignedUrl( + Consumer presignedDownloadFileRequest) { + return downloadFileWithPresignedUrl( + PresignedDownloadFileRequest.builder().applyMutation(presignedDownloadFileRequest).build()); + } + + /** + * Downloads an object using a pre-signed URL through the given {@link AsyncResponseTransformer}. For downloading + * to a file, you may use {@link #downloadFileWithPresignedUrl(PresignedDownloadFileRequest)} instead. + *

+ * This method supports multipart downloads when using a CRT-based or multipart-enabled S3 client, + * providing enhanced throughput and reliability for large objects. Progress can be monitored + * through {@link TransferListener}s attached to the request. + *

+ * Note: The result of the operation doesn't support pause and resume functionality. + *

+ *

+ * Usage Example (downloading to memory - not suitable for large objects): + * {@snippet : + * S3TransferManager transferManager = S3TransferManager.create(); + * + * // Create presigned URL (typically done by another service) + * PresignedUrlDownloadRequest presignedRequest = PresignedUrlDownloadRequest.builder() + * .presignedUrl(presignedUrl) + * .build(); + * + * PresignedDownloadRequest> request = + * PresignedDownloadRequest.builder() + * .presignedUrlDownloadRequest(presignedRequest) + * .responseTransformer(AsyncResponseTransformer.toBytes()) + * .addTransferListener(LoggingTransferListener.create()) + * .build(); + * + * Download> download = transferManager.downloadWithPresignedUrl(request); + * ResponseBytes result = download.completionFuture().join().result(); + * } + * + * @param presignedDownloadRequest the presigned download request + * @param The type of data the {@link AsyncResponseTransformer} produces + * @return A {@link Download} that can be used to track the ongoing transfer + * @see #downloadFileWithPresignedUrl(PresignedDownloadFileRequest) + * @see AsyncResponseTransformer + */ + default Download downloadWithPresignedUrl( + PresignedDownloadRequest presignedDownloadRequest) { + throw new UnsupportedOperationException(); + } + /** * Create an {@code S3TransferManager} using the default values. *

diff --git a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/DelegatingS3TransferManager.java b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/DelegatingS3TransferManager.java index f2929bf8f988..15d309552857 100644 --- a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/DelegatingS3TransferManager.java +++ b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/DelegatingS3TransferManager.java @@ -27,6 +27,8 @@ import software.amazon.awssdk.transfer.s3.model.DownloadRequest; import software.amazon.awssdk.transfer.s3.model.FileDownload; import software.amazon.awssdk.transfer.s3.model.FileUpload; +import software.amazon.awssdk.transfer.s3.model.PresignedDownloadFileRequest; +import software.amazon.awssdk.transfer.s3.model.PresignedDownloadRequest; import software.amazon.awssdk.transfer.s3.model.ResumableFileDownload; import software.amazon.awssdk.transfer.s3.model.Upload; import software.amazon.awssdk.transfer.s3.model.UploadDirectoryRequest; @@ -85,6 +87,16 @@ public Copy copy(CopyRequest copyRequest) { return delegate.copy(copyRequest); } + @Override + public FileDownload downloadFileWithPresignedUrl(PresignedDownloadFileRequest presignedDownloadFileRequest) { + return delegate.downloadFileWithPresignedUrl(presignedDownloadFileRequest); + } + + @Override + public Download downloadWithPresignedUrl(PresignedDownloadRequest presignedDownloadRequest) { + return delegate.downloadWithPresignedUrl(presignedDownloadRequest); + } + @Override public void close() { delegate.close(); diff --git a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/GenericS3TransferManager.java b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/GenericS3TransferManager.java index 4814b862e379..8d8825c67eac 100644 --- a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/GenericS3TransferManager.java +++ b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/GenericS3TransferManager.java @@ -75,6 +75,8 @@ import software.amazon.awssdk.transfer.s3.model.DownloadRequest; import software.amazon.awssdk.transfer.s3.model.FileDownload; import software.amazon.awssdk.transfer.s3.model.FileUpload; +import software.amazon.awssdk.transfer.s3.model.PresignedDownloadFileRequest; +import software.amazon.awssdk.transfer.s3.model.PresignedDownloadRequest; import software.amazon.awssdk.transfer.s3.model.ResumableFileDownload; import software.amazon.awssdk.transfer.s3.model.ResumableFileUpload; import software.amazon.awssdk.transfer.s3.model.Upload; @@ -597,6 +599,80 @@ public final Copy copy(CopyRequest copyRequest) { return new DefaultCopy(returnFuture, progressUpdater.progress()); } + @Override + public final FileDownload downloadFileWithPresignedUrl(PresignedDownloadFileRequest presignedDownloadFileRequest) { + Validate.paramNotNull(presignedDownloadFileRequest, "presignedDownloadFileRequest"); + + AsyncResponseTransformer responseTransformer = + AsyncResponseTransformer.toFile(presignedDownloadFileRequest.destination(), + FileTransformerConfiguration.defaultCreateOrReplaceExisting()); + + CompletableFuture returnFuture = new CompletableFuture<>(); + + TransferProgressUpdater progressUpdater = new TransferProgressUpdater(presignedDownloadFileRequest, null); + progressUpdater.transferInitiated(); + + responseTransformer = isS3ClientMultipartEnabled() + ? progressUpdater.wrapForNonSerialFileDownload( + responseTransformer, GetObjectRequest.builder().build()) + : progressUpdater.wrapResponseTransformer(responseTransformer); + progressUpdater.registerCompletion(returnFuture); + + try { + CompletableFuture future = s3AsyncClient.presignedUrlExtension().getObject( + presignedDownloadFileRequest.presignedUrlDownloadRequest(), responseTransformer); + + CompletableFutureUtils.forwardExceptionTo(returnFuture, future); + CompletableFutureUtils.forwardTransformedResultTo(future, returnFuture, + res -> CompletedFileDownload.builder() + .response(res) + .build()); + } catch (Throwable throwable) { + returnFuture.completeExceptionally(throwable); + } + + return new DefaultFileDownload(returnFuture, progressUpdater.progress(), + () -> { + throw new UnsupportedOperationException( + "Pause is not supported for presigned URL downloads"); + }, null); + } + + @Override + public final Download downloadWithPresignedUrl( + PresignedDownloadRequest presignedDownloadRequest) { + Validate.paramNotNull(presignedDownloadRequest, "presignedDownloadRequest"); + + AsyncResponseTransformer responseTransformer = + presignedDownloadRequest.responseTransformer(); + + CompletableFuture> returnFuture = new CompletableFuture<>(); + + TransferProgressUpdater progressUpdater = new TransferProgressUpdater(presignedDownloadRequest, null); + progressUpdater.transferInitiated(); + + responseTransformer = isS3ClientMultipartEnabled() + ? progressUpdater.wrapForNonSerialFileDownload( + responseTransformer, GetObjectRequest.builder().build()) + : progressUpdater.wrapResponseTransformer(responseTransformer); + progressUpdater.registerCompletion(returnFuture); + + try { + CompletableFuture future = s3AsyncClient.presignedUrlExtension().getObject( + presignedDownloadRequest.presignedUrlDownloadRequest(), responseTransformer); + + CompletableFutureUtils.forwardExceptionTo(returnFuture, future); + CompletableFutureUtils.forwardTransformedResultTo(future, returnFuture, + r -> CompletedDownload.builder() + .result(r) + .build()); + } catch (Throwable throwable) { + returnFuture.completeExceptionally(throwable); + } + + return new DefaultDownload<>(returnFuture, progressUpdater.progress()); + } + @Override public final void close() { if (isDefaultS3AsyncClient) { diff --git a/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/S3TransferManagerPresignedUrlDownloadTest.java b/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/S3TransferManagerPresignedUrlDownloadTest.java new file mode 100644 index 000000000000..16de24bec605 --- /dev/null +++ b/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/S3TransferManagerPresignedUrlDownloadTest.java @@ -0,0 +1,192 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.transfer.s3.internal; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import java.net.URL; +import java.nio.file.Paths; +import java.util.concurrent.CompletableFuture; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import software.amazon.awssdk.core.ResponseBytes; +import software.amazon.awssdk.core.async.AsyncResponseTransformer; +import software.amazon.awssdk.services.s3.S3AsyncClient; +import software.amazon.awssdk.services.s3.model.GetObjectResponse; +import software.amazon.awssdk.services.s3.presignedurl.AsyncPresignedUrlExtension; +import software.amazon.awssdk.services.s3.presignedurl.model.PresignedUrlDownloadRequest; +import software.amazon.awssdk.transfer.s3.model.CompletedDownload; +import software.amazon.awssdk.transfer.s3.model.CompletedFileDownload; +import software.amazon.awssdk.transfer.s3.model.FileDownload; +import software.amazon.awssdk.transfer.s3.model.PresignedDownloadFileRequest; +import software.amazon.awssdk.transfer.s3.model.PresignedDownloadRequest; + +/** + * Unit tests for S3TransferManager presigned URL download functionality. + */ +class S3TransferManagerPresignedUrlDownloadTest { + private static final String PRESIGNED_URL = "https://test-bucket.s3.amazonaws.com/test-key" + + "?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=AKID" + + "&X-Amz-Date=20260101T000000Z&X-Amz-Expires=600" + + "&X-Amz-SignedHeaders=host&X-Amz-Signature=abc123"; + + private S3AsyncClient mockS3AsyncClient; + private AsyncPresignedUrlExtension mockPresignedUrlExtension; + private GenericS3TransferManager tm; + private URL presignedUrl; + + @BeforeEach + public void methodSetup() throws Exception { + mockS3AsyncClient = mock(S3AsyncClient.class); + mockPresignedUrlExtension = mock(AsyncPresignedUrlExtension.class); + presignedUrl = new URL(PRESIGNED_URL); + + when(mockS3AsyncClient.presignedUrlExtension()).thenReturn(mockPresignedUrlExtension); + + tm = new GenericS3TransferManager(mockS3AsyncClient, + mock(UploadDirectoryHelper.class), + mock(TransferManagerConfiguration.class), + mock(DownloadDirectoryHelper.class)); + } + + @AfterEach + public void methodTeardown() { + tm.close(); + } + + @Test + void downloadFileWithPresignedUrl_withValidRequest_returnsResponse() { + GetObjectResponse response = GetObjectResponse.builder().build(); + stubGetObject(CompletableFuture.completedFuture(response)); + + CompletedFileDownload completed = tm.downloadFileWithPresignedUrl(fileDownloadRequest()) + .completionFuture().join(); + + assertThat(completed.response()).isEqualTo(response); + } + + @Test + void downloadWithPresignedUrl_withValidRequest_returnsResponse() { + ResponseBytes responseBytes = ResponseBytes.fromByteArray( + GetObjectResponse.builder().build(), "test".getBytes()); + stubGetObject(CompletableFuture.completedFuture(responseBytes)); + + CompletedDownload> completed = + tm.downloadWithPresignedUrl(bytesDownloadRequest()).completionFuture().join(); + + assertThat(completed.result()).isEqualTo(responseBytes); + } + + @Test + void downloadFileWithPresignedUrl_withConsumerBuilder_returnsResponse() { + GetObjectResponse response = GetObjectResponse.builder().build(); + stubGetObject(CompletableFuture.completedFuture(response)); + + CompletedFileDownload completed = tm.downloadFileWithPresignedUrl( + request -> request.presignedUrlDownloadRequest(p -> p.presignedUrl(presignedUrl)) + .destination(Paths.get("/tmp/test"))) + .completionFuture().join(); + + assertThat(completed.response()).isEqualTo(response); + } + + @Test + void downloadFileWithPresignedUrl_whenCancelled_shouldForwardCancellation() { + CompletableFuture s3Future = new CompletableFuture<>(); + stubGetObject(s3Future); + + CompletableFuture future = + tm.downloadFileWithPresignedUrl(fileDownloadRequest()).completionFuture(); + + future.cancel(true); + assertThat(s3Future).isCancelled(); + } + + @Test + void downloadFileWithPresignedUrl_whenRequestFails_shouldCompleteExceptionally() { + CompletableFuture failedFuture = new CompletableFuture<>(); + failedFuture.completeExceptionally(new RuntimeException("download failed")); + stubGetObject(failedFuture); + + assertThatThrownBy(() -> tm.downloadFileWithPresignedUrl(fileDownloadRequest()).completionFuture().join()) + .hasCauseInstanceOf(RuntimeException.class) + .hasMessageContaining("download failed"); + } + + @Test + void downloadWithPresignedUrl_whenCancelled_shouldForwardCancellation() { + CompletableFuture s3Future = new CompletableFuture<>(); + stubGetObject(s3Future); + + CompletableFuture>> future = + tm.downloadWithPresignedUrl(bytesDownloadRequest()).completionFuture(); + + future.cancel(true); + assertThat(s3Future).isCancelled(); + } + + @Test + void downloadFileWithPresignedUrl_withNullRequest_shouldThrowNullPointerException() { + assertThatThrownBy(() -> tm.downloadFileWithPresignedUrl((PresignedDownloadFileRequest) null)) + .isInstanceOf(NullPointerException.class); + } + + @Test + void downloadWithPresignedUrl_withNullRequest_shouldThrowNullPointerException() { + assertThatThrownBy(() -> tm.downloadWithPresignedUrl(null)) + .isInstanceOf(NullPointerException.class); + } + @Test + void downloadFileWithPresignedUrl_pause_shouldThrowUnsupportedOperationException() { + GetObjectResponse response = GetObjectResponse.builder().build(); + stubGetObject(CompletableFuture.completedFuture(response)); + + FileDownload download = tm.downloadFileWithPresignedUrl(fileDownloadRequest()); + download.completionFuture().join(); + + assertThatThrownBy(download::pause) + .isInstanceOf(UnsupportedOperationException.class) + .hasMessageContaining("Pause is not supported for presigned URL downloads"); + } + + private PresignedDownloadFileRequest fileDownloadRequest() { + return PresignedDownloadFileRequest.builder() + .presignedUrlDownloadRequest(PresignedUrlDownloadRequest.builder() + .presignedUrl(presignedUrl) + .build()) + .destination(Paths.get("/tmp/test")) + .build(); + } + + private PresignedDownloadRequest> bytesDownloadRequest() { + return PresignedDownloadRequest.builder() + .presignedUrlDownloadRequest(PresignedUrlDownloadRequest.builder() + .presignedUrl(presignedUrl) + .build()) + .responseTransformer(AsyncResponseTransformer.toBytes()) + .build(); + } + + private void stubGetObject(CompletableFuture future) { + when(mockPresignedUrlExtension.getObject(any(PresignedUrlDownloadRequest.class), any(AsyncResponseTransformer.class))) + .thenReturn(future); + } +} From a7fdaa4f4117a4b35506f8543068520fb8efbd8e Mon Sep 17 00:00:00 2001 From: jencymaryjoseph <35571282+jencymaryjoseph@users.noreply.github.com> Date: Mon, 18 May 2026 12:25:01 -0700 Subject: [PATCH 18/30] Fix presigned url multipart download for empty objects (#6952) --- ...dUrlExtensionMultipartIntegrationTest.java | 35 ++++++++ .../AsyncPresignedUrlExtensionTestSuite.java | 40 ++++++++++ ...ignedUrlMultipartDownloaderSubscriber.java | 16 +++- .../multipart/PresignedUrlDownloadHelper.java | 50 +++++++++++- ...ignedUrlMultipartDownloaderSubscriber.java | 11 ++- ...dUrlMultipartDownloaderSubscriberTest.java | 66 +++++++++++++++- ...ipartDownloaderSubscriberWiremockTest.java | 79 +++++++++++++++++++ 7 files changed, 290 insertions(+), 7 deletions(-) create mode 100644 services/s3/src/it/java/software/amazon/awssdk/services/s3/presignedurl/AsyncPresignedUrlExtensionMultipartIntegrationTest.java diff --git a/services/s3/src/it/java/software/amazon/awssdk/services/s3/presignedurl/AsyncPresignedUrlExtensionMultipartIntegrationTest.java b/services/s3/src/it/java/software/amazon/awssdk/services/s3/presignedurl/AsyncPresignedUrlExtensionMultipartIntegrationTest.java new file mode 100644 index 000000000000..16e41325817f --- /dev/null +++ b/services/s3/src/it/java/software/amazon/awssdk/services/s3/presignedurl/AsyncPresignedUrlExtensionMultipartIntegrationTest.java @@ -0,0 +1,35 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ +package software.amazon.awssdk.services.s3.presignedurl; + +import org.junit.jupiter.api.BeforeAll; +import software.amazon.awssdk.services.s3.S3AsyncClient; + + +public class AsyncPresignedUrlExtensionMultipartIntegrationTest extends AsyncPresignedUrlExtensionTestSuite { + + @BeforeAll + static void setUpIntegrationTest() { + S3AsyncClient s3AsyncClient = s3AsyncClientBuilder() + .multipartEnabled(true) + .build(); + presignedUrlExtension = s3AsyncClient.presignedUrlExtension(); + } + + @Override + protected S3AsyncClient createS3AsyncClient() { + return s3AsyncClientBuilder().build(); + } +} diff --git a/services/s3/src/it/java/software/amazon/awssdk/services/s3/presignedurl/AsyncPresignedUrlExtensionTestSuite.java b/services/s3/src/it/java/software/amazon/awssdk/services/s3/presignedurl/AsyncPresignedUrlExtensionTestSuite.java index eee244ca3485..d32c7038317a 100644 --- a/services/s3/src/it/java/software/amazon/awssdk/services/s3/presignedurl/AsyncPresignedUrlExtensionTestSuite.java +++ b/services/s3/src/it/java/software/amazon/awssdk/services/s3/presignedurl/AsyncPresignedUrlExtensionTestSuite.java @@ -16,6 +16,7 @@ import static org.apache.commons.lang3.RandomStringUtils.randomAscii; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.AssertionsForClassTypes.assertThatThrownBy; import java.io.ByteArrayInputStream; import java.net.URL; @@ -47,6 +48,7 @@ import software.amazon.awssdk.services.s3.model.DeleteObjectRequest; import software.amazon.awssdk.services.s3.model.GetObjectResponse; import software.amazon.awssdk.services.s3.model.PutObjectRequest; +import software.amazon.awssdk.services.s3.model.S3Exception; import software.amazon.awssdk.services.s3.presigner.S3Presigner; import software.amazon.awssdk.services.s3.presigner.model.PresignedGetObjectRequest; import software.amazon.awssdk.services.s3.presignedurl.model.PresignedUrlDownloadRequest; @@ -240,6 +242,44 @@ public void close() {} } } + @Test + void getObject_emptyObject_toBytes_shouldSucceed() throws Exception { + String emptyKey = uploadTestObject("empty-object-bytes", ""); + + PresignedUrlDownloadRequest request = createRequestForKey(emptyKey); + ResponseBytes response = + presignedUrlExtension.getObject(request, AsyncResponseTransformer.toBytes()) + .get(30, TimeUnit.SECONDS); + + assertThat(response.asByteArray()).isEmpty(); + assertThat(response.response().contentLength()).isEqualTo(0L); + } + + @Test + void getObject_emptyObject_toFile_shouldSucceed() throws Exception { + String emptyKey = uploadTestObject("empty-object-file", ""); + + PresignedUrlDownloadRequest request = createRequestForKey(emptyKey); + Path downloadFile = temporaryFolder.resolve("empty-download-" + UUID.randomUUID() + ".bin"); + GetObjectResponse response = + presignedUrlExtension.getObject(request, downloadFile) + .get(30, TimeUnit.SECONDS); + + assertThat(downloadFile).exists(); + assertThat(downloadFile.toFile().length()).isEqualTo(0L); + } + + @Test + void getObject_emptyObject_withRange_shouldThrow416() throws Exception { + String emptyKey = uploadTestObject("empty-object-range", ""); + + PresignedUrlDownloadRequest request = createRequestForKey(emptyKey, "bytes=0-1024"); + + assertThatThrownBy(() -> presignedUrlExtension.getObject(request, AsyncResponseTransformer.toBytes()) + .get(30, TimeUnit.SECONDS)) + .hasCauseInstanceOf(S3Exception.class); + } + static Stream basicFunctionalityTestData() { return Stream.of( Arguments.of("getObject_withValidUrl_returnsContent", diff --git a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/ParallelPresignedUrlMultipartDownloaderSubscriber.java b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/ParallelPresignedUrlMultipartDownloaderSubscriber.java index bf1e9ddaf6ab..8df801f96d34 100644 --- a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/ParallelPresignedUrlMultipartDownloaderSubscriber.java +++ b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/ParallelPresignedUrlMultipartDownloaderSubscriber.java @@ -138,13 +138,25 @@ private void sendFirstRequest(AsyncResponseTransformer { inFlightRequests.remove(0); inFlightPermits.release(); - if (error != null || isCompletedExceptionally.get()) { + if (error != null) { + if (PresignedUrlDownloadHelper.isRangeNotSatisfiable(error)) { + resultFuture.completeExceptionally( + new PresignedUrlDownloadHelper.EmptyObjectRangeNotSatisfiableException(error)); + synchronized (subscriptionLock) { + subscription.cancel(); + } + } else { + handlePartError(error, 0); + } + return; + } + + if (isCompletedExceptionally.get()) { handlePartError(error, 0); return; } diff --git a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/PresignedUrlDownloadHelper.java b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/PresignedUrlDownloadHelper.java index f62ec1d7d202..3f5c955abad7 100644 --- a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/PresignedUrlDownloadHelper.java +++ b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/PresignedUrlDownloadHelper.java @@ -16,12 +16,14 @@ package software.amazon.awssdk.services.s3.internal.multipart; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionException; import software.amazon.awssdk.annotations.SdkInternalApi; import software.amazon.awssdk.core.SplittingTransformerConfiguration; import software.amazon.awssdk.core.async.AsyncResponseTransformer; import software.amazon.awssdk.core.exception.SdkClientException; import software.amazon.awssdk.services.s3.S3AsyncClient; import software.amazon.awssdk.services.s3.model.GetObjectResponse; +import software.amazon.awssdk.services.s3.model.S3Exception; import software.amazon.awssdk.services.s3.presignedurl.AsyncPresignedUrlExtension; import software.amazon.awssdk.services.s3.presignedurl.model.PresignedUrlDownloadRequest; import software.amazon.awssdk.utils.Logger; @@ -60,6 +62,33 @@ public CompletableFuture downloadObject( return asyncPresignedUrlExtension.getObject(presignedRequest, asyncResponseTransformer); } + CompletableFuture resultFuture = new CompletableFuture<>(); + doMultipartDownload(presignedRequest, asyncResponseTransformer) + .whenComplete((result, error) -> { + Throwable cause = error instanceof CompletionException ? error.getCause() : error; + if (cause instanceof EmptyObjectRangeNotSatisfiableException) { + log.debug(() -> "Received 416 on first request, falling back to non-range GET for empty object"); + asyncPresignedUrlExtension.getObject(presignedRequest, asyncResponseTransformer) + .whenComplete((r, e) -> { + if (e != null) { + resultFuture.completeExceptionally(e); + } else { + resultFuture.complete(r); + } + }); + } else if (error != null) { + resultFuture.completeExceptionally(error); + } else { + resultFuture.complete(result); + } + }); + return resultFuture; + } + + private CompletableFuture doMultipartDownload( + PresignedUrlDownloadRequest presignedRequest, + AsyncResponseTransformer asyncResponseTransformer) { + SplittingTransformerConfiguration splittingConfig = SplittingTransformerConfiguration.builder() .bufferSizeInBytes(bufferSizeInBytes) .build(); @@ -112,4 +141,23 @@ static SdkClientException missingContentRangeHeader() { static SdkClientException invalidContentLength() { return SdkClientException.create("Invalid or missing Content-Length in response"); } -} + + /** + * Returns true if the error is a 416 Range Not Satisfiable response from S3. + * Used by subscribers to detect empty object responses on the first range request. + */ + static boolean isRangeNotSatisfiable(Throwable error) { + Throwable cause = error instanceof CompletionException ? error.getCause() : error; + return cause instanceof S3Exception && ((S3Exception) cause).statusCode() == 416; + } + + /** + * Marker exception wrapping a 416 on the first range request, signaling an empty object. + * Used to distinguish from 416 errors on subsequent requests which should propagate as failures. + */ + static class EmptyObjectRangeNotSatisfiableException extends RuntimeException { + EmptyObjectRangeNotSatisfiableException(Throwable cause) { + super("Object is empty (416 on first range request)", cause); + } + } +} \ No newline at end of file diff --git a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/PresignedUrlMultipartDownloaderSubscriber.java b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/PresignedUrlMultipartDownloaderSubscriber.java index d76f0a8e8e6b..a4b4326b464b 100644 --- a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/PresignedUrlMultipartDownloaderSubscriber.java +++ b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/PresignedUrlMultipartDownloaderSubscriber.java @@ -130,7 +130,16 @@ private void makeRangeRequest(int partIndex, .getObject(partRequest, asyncResponseTransformer) .whenComplete((response, error) -> { if (error != null) { - handleError(error); + if (partIndex == 0 && PresignedUrlDownloadHelper.isRangeNotSatisfiable(error)) { + log.debug(() -> "Received 416 on first range request, object is empty"); + resultFuture.completeExceptionally( + new PresignedUrlDownloadHelper.EmptyObjectRangeNotSatisfiableException(error)); + synchronized (lock) { + subscription.cancel(); + } + } else { + handleError(error); + } return; } if (validatePart(response, partIndex, asyncResponseTransformer)) { diff --git a/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/multipart/ParallelPresignedUrlMultipartDownloaderSubscriberTest.java b/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/multipart/ParallelPresignedUrlMultipartDownloaderSubscriberTest.java index f24b2b3a2d1b..e29558531418 100644 --- a/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/multipart/ParallelPresignedUrlMultipartDownloaderSubscriberTest.java +++ b/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/multipart/ParallelPresignedUrlMultipartDownloaderSubscriberTest.java @@ -16,7 +16,7 @@ package software.amazon.awssdk.services.s3.internal.multipart; import static com.github.tomakehurst.wiremock.client.WireMock.aResponse; -import static com.github.tomakehurst.wiremock.client.WireMock.findAll; +import static com.github.tomakehurst.wiremock.client.WireMock.absent; import static com.github.tomakehurst.wiremock.client.WireMock.get; import static com.github.tomakehurst.wiremock.client.WireMock.getRequestedFor; import static com.github.tomakehurst.wiremock.client.WireMock.matching; @@ -28,7 +28,6 @@ import com.github.tomakehurst.wiremock.junit5.WireMockRuntimeInfo; import com.github.tomakehurst.wiremock.junit5.WireMockTest; -import com.github.tomakehurst.wiremock.verification.LoggedRequest; import java.io.IOException; import java.net.MalformedURLException; import java.net.URI; @@ -37,7 +36,6 @@ import java.nio.file.Files; import java.nio.file.Path; import java.util.Arrays; -import java.util.List; import java.util.UUID; import java.util.concurrent.CompletionException; import org.junit.jupiter.api.AfterEach; @@ -276,6 +274,68 @@ void multiPartDownload_manyParts_shouldCompleteSuccessfully() throws Exception { verify(13, getRequestedFor(urlEqualTo(PRESIGNED_URL_PATH))); } + @Test + void emptyObject_416OnFirstRequest_shouldFallbackToNonRangeGet() { + stubFor(get(urlEqualTo(PRESIGNED_URL_PATH)) + .withHeader("Range", matching("bytes=.*")) + .willReturn(aResponse() + .withStatus(416) + .withBody("InvalidRange" + + "The requested range is not satisfiable"))); + stubFor(get(urlEqualTo(PRESIGNED_URL_PATH)) + .withHeader("Range", absent()) + .willReturn(aResponse() + .withStatus(200) + .withHeader("Content-Length", "0") + .withHeader("ETag", "\"empty-etag\"") + .withBody(new byte[0]))); + + tempFile = createTempFileUnchecked(); + PresignedUrlDownloadRequest request = PresignedUrlDownloadRequest.builder() + .presignedUrl(presignedUrl) + .build(); + + s3AsyncClient.presignedUrlExtension() + .getObject(request, AsyncResponseTransformer.toFile(tempFile)) + .join(); + + assertThat(tempFile.toFile()).exists(); + assertThat(tempFile.toFile().length()).isEqualTo(0L); + } + + @Test + void multipartObject_416OnSecondRequest_shouldFailWithError() { + stubFor(get(urlEqualTo(PRESIGNED_URL_PATH)) + .inScenario("416-on-second") + .whenScenarioStateIs("Started") + .withHeader("Range", matching("bytes=0-15")) + .willReturn(aResponse() + .withStatus(206) + .withHeader("Content-Length", "16") + .withHeader("Content-Range", "bytes 0-15/32") + .withHeader("ETag", "\"test-etag\"") + .withBody(Arrays.copyOfRange(TEST_DATA, 0, 16))) + .willSetStateTo("first-done")); + + stubFor(get(urlEqualTo(PRESIGNED_URL_PATH)) + .inScenario("416-on-second") + .whenScenarioStateIs("first-done") + .willReturn(aResponse() + .withStatus(416) + .withBody("InvalidRange" + + "The requested range is not satisfiable"))); + + tempFile = createTempFileUnchecked(); + PresignedUrlDownloadRequest request = PresignedUrlDownloadRequest.builder() + .presignedUrl(presignedUrl) + .build(); + + assertThatThrownBy(() -> s3AsyncClient.presignedUrlExtension() + .getObject(request, AsyncResponseTransformer.toFile(tempFile)) + .join()) + .isInstanceOf(CompletionException.class); + } + private static Path createTempFile() throws IOException { Path path = Files.createTempFile("parallel-test-" + UUID.randomUUID(), ".tmp"); Files.deleteIfExists(path); diff --git a/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/multipart/PresignedUrlMultipartDownloaderSubscriberWiremockTest.java b/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/multipart/PresignedUrlMultipartDownloaderSubscriberWiremockTest.java index 2ff0e4da46b7..90875fa524be 100644 --- a/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/multipart/PresignedUrlMultipartDownloaderSubscriberWiremockTest.java +++ b/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/multipart/PresignedUrlMultipartDownloaderSubscriberWiremockTest.java @@ -16,6 +16,7 @@ package software.amazon.awssdk.services.s3.internal.multipart; import static com.github.tomakehurst.wiremock.client.WireMock.aResponse; +import static com.github.tomakehurst.wiremock.client.WireMock.absent; import static com.github.tomakehurst.wiremock.client.WireMock.get; import static com.github.tomakehurst.wiremock.client.WireMock.stubFor; import static com.github.tomakehurst.wiremock.client.WireMock.urlEqualTo; @@ -260,6 +261,84 @@ void onNext_withNullTransformer_shouldThrowException() { .hasMessageContaining("onNext must not be called with null asyncResponseTransformer"); } + @ParameterizedTest(name = "presignedUrlDownload_emptyObject_shouldFallbackToNonRangeGet [{0}]") + @MethodSource("transformerTypes") + @SuppressWarnings("unchecked") + void presignedUrlDownload_emptyObject_shouldFallbackToNonRangeGet(String transformerType) throws IOException { + stubFor(get(urlEqualTo(PRESIGNED_URL_PATH)) + .withHeader("Range", matching("bytes=.*")) + .willReturn(aResponse() + .withStatus(416) + .withBody("InvalidRange" + + "The requested range is not satisfiable"))); + stubFor(get(urlEqualTo(PRESIGNED_URL_PATH)) + .withHeader("Range", absent()) + .willReturn(aResponse() + .withStatus(200) + .withHeader("Content-Length", "0") + .withHeader("ETag", "\"empty-etag\"") + .withBody(new byte[0]))); + + PresignedUrlDownloadRequest request = PresignedUrlDownloadRequest.builder() + .presignedUrl(presignedUrl) + .build(); + Object result = executeDownload(request, transformerType).join(); + if ("toBytes".equals(transformerType)) { + ResponseBytes bytes = (ResponseBytes) result; + assertThat(bytes.asByteArray()).isEmpty(); + } else { + assertThat(tempFile.toFile()).exists(); + assertThat(Files.size(tempFile)).isEqualTo(0L); + } + } + + @ParameterizedTest(name = "presignedUrlDownload_416OnSecondRequest_shouldFailWithError [{0}]") + @MethodSource("transformerTypes") + void presignedUrlDownload_416OnSecondRequest_shouldFailWithError(String transformerType) { + stubFor(get(urlEqualTo(PRESIGNED_URL_PATH)) + .inScenario("416-on-second") + .whenScenarioStateIs("Started") + .withHeader("Range", matching("bytes=0-15")) + .willReturn(aResponse() + .withStatus(206) + .withHeader("Content-Length", "16") + .withHeader("Content-Range", "bytes 0-15/32") + .withHeader("ETag", "\"test-etag\"") + .withBody(Arrays.copyOfRange(TEST_DATA, 0, 16))) + .willSetStateTo("first-done")); + + stubFor(get(urlEqualTo(PRESIGNED_URL_PATH)) + .inScenario("416-on-second") + .whenScenarioStateIs("first-done") + .willReturn(aResponse() + .withStatus(416) + .withBody("InvalidRange" + + "The requested range is not satisfiable"))); + + PresignedUrlDownloadRequest request = PresignedUrlDownloadRequest.builder() + .presignedUrl(presignedUrl) + .build(); + assertThatThrownBy(() -> executeDownload(request, transformerType).join()) + .hasRootCauseInstanceOf(S3Exception.class); + } + + @ParameterizedTest(name = "presignedUrlDownload_withRangeHeader_emptyObject_shouldThrow416 [{0}]") + @MethodSource("transformerTypes") + void presignedUrlDownload_withRangeHeader_emptyObject_shouldThrow416(String transformerType) { + stubFor(get(urlEqualTo(PRESIGNED_URL_PATH)) + .willReturn(aResponse() + .withStatus(416) + .withBody("InvalidRange" + + "The requested range is not satisfiable"))); + + PresignedUrlDownloadRequest request = PresignedUrlDownloadRequest.builder() + .presignedUrl(presignedUrl) + .range("bytes=0-1024") + .build(); + assertThatThrownBy(() -> executeDownload(request, transformerType).join()) + .hasRootCauseInstanceOf(S3Exception.class); + } + @AfterEach void cleanup() { if (tempFile != null && Files.exists(tempFile)) { From 12af83dea2c2564483b5cfe45f18845fed4b6da8 Mon Sep 17 00:00:00 2001 From: jencymaryjoseph <35571282+jencymaryjoseph@users.noreply.github.com> Date: Wed, 20 May 2026 16:51:29 -0700 Subject: [PATCH 19/30] Add retry tests and onError cancel fix for presigned URL multipart downloads (#6985) --- ...ignedUrlMultipartDownloaderSubscriber.java | 52 ++- ...DownloaderSubscriberRetryWiremockTest.java | 357 ++++++++++++++++++ ...ipartDownloaderSubscriberWiremockTest.java | 2 +- 3 files changed, 390 insertions(+), 21 deletions(-) create mode 100644 services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/multipart/PresignedUrlMultipartDownloaderSubscriberRetryWiremockTest.java diff --git a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/PresignedUrlMultipartDownloaderSubscriber.java b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/PresignedUrlMultipartDownloaderSubscriber.java index a4b4326b464b..2f698eea58f2 100644 --- a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/PresignedUrlMultipartDownloaderSubscriber.java +++ b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/PresignedUrlMultipartDownloaderSubscriber.java @@ -16,7 +16,9 @@ package software.amazon.awssdk.services.s3.internal.multipart; import java.util.Optional; +import java.util.Queue; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.atomic.AtomicInteger; import org.reactivestreams.Subscriber; import org.reactivestreams.Subscription; @@ -71,6 +73,11 @@ public class PresignedUrlMultipartDownloaderSubscriber private final AtomicInteger completedParts; private final AtomicInteger requestsSent; + /** + * Store the GetObject futures so we can cancel them if onError() is invoked. + */ + private final Queue> getObjectFutures = new ConcurrentLinkedQueue<>(); + private volatile Long totalContentLength; private volatile Integer totalParts; private volatile String eTag; @@ -126,26 +133,27 @@ private void makeRangeRequest(int partIndex, log.debug(() -> "Sending range request for part " + partIndex + " with range=" + partRequest.range()); requestsSent.incrementAndGet(); - s3AsyncClient.presignedUrlExtension() - .getObject(partRequest, asyncResponseTransformer) - .whenComplete((response, error) -> { - if (error != null) { - if (partIndex == 0 && PresignedUrlDownloadHelper.isRangeNotSatisfiable(error)) { - log.debug(() -> "Received 416 on first range request, object is empty"); - resultFuture.completeExceptionally( - new PresignedUrlDownloadHelper.EmptyObjectRangeNotSatisfiableException(error)); - synchronized (lock) { - subscription.cancel(); - } - } else { - handleError(error); - } - return; - } - if (validatePart(response, partIndex, asyncResponseTransformer)) { - requestMoreIfNeeded(completedParts.get()); - } - }); + CompletableFuture responseFuture = s3AsyncClient.presignedUrlExtension() + .getObject(partRequest, asyncResponseTransformer); + getObjectFutures.add(responseFuture); + responseFuture.whenComplete((response, error) -> { + if (error != null) { + if (partIndex == 0 && PresignedUrlDownloadHelper.isRangeNotSatisfiable(error)) { + log.debug(() -> "Received 416 on first range request, object is empty"); + resultFuture.completeExceptionally( + new PresignedUrlDownloadHelper.EmptyObjectRangeNotSatisfiableException(error)); + synchronized (lock) { + subscription.cancel(); + } + } else { + handleError(error); + } + return; + } + if (validatePart(response, partIndex, asyncResponseTransformer)) { + requestMoreIfNeeded(completedParts.get()); + } + }); } private boolean validatePart(GetObjectResponse response, int partIndex, @@ -292,6 +300,10 @@ private void handleError(Throwable t) { @Override public void onError(Throwable t) { log.debug(() -> "Error in multipart download", t); + CompletableFuture partFuture; + while ((partFuture = getObjectFutures.poll()) != null) { + partFuture.cancel(true); + } future.completeExceptionally(t); if (resultFuture != null) { resultFuture.completeExceptionally(t); diff --git a/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/multipart/PresignedUrlMultipartDownloaderSubscriberRetryWiremockTest.java b/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/multipart/PresignedUrlMultipartDownloaderSubscriberRetryWiremockTest.java new file mode 100644 index 000000000000..fe81ea43715b --- /dev/null +++ b/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/multipart/PresignedUrlMultipartDownloaderSubscriberRetryWiremockTest.java @@ -0,0 +1,357 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.services.s3.internal.multipart; + +import static com.github.tomakehurst.wiremock.client.WireMock.aResponse; +import static com.github.tomakehurst.wiremock.client.WireMock.exactly; +import static com.github.tomakehurst.wiremock.client.WireMock.get; +import static com.github.tomakehurst.wiremock.client.WireMock.getRequestedFor; +import static com.github.tomakehurst.wiremock.client.WireMock.matching; +import static com.github.tomakehurst.wiremock.client.WireMock.moreThan; +import static com.github.tomakehurst.wiremock.client.WireMock.stubFor; +import static com.github.tomakehurst.wiremock.client.WireMock.urlEqualTo; +import static com.github.tomakehurst.wiremock.client.WireMock.verify; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.junit.jupiter.api.Assertions.assertArrayEquals; + +import com.github.tomakehurst.wiremock.http.Fault; +import com.github.tomakehurst.wiremock.junit5.WireMockRuntimeInfo; +import com.github.tomakehurst.wiremock.junit5.WireMockTest; +import com.github.tomakehurst.wiremock.stubbing.Scenario; +import java.io.IOException; +import java.net.MalformedURLException; +import java.net.URI; +import java.net.URL; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Arrays; +import java.util.UUID; +import java.util.concurrent.CompletionException; +import java.util.concurrent.TimeUnit; +import java.util.stream.Stream; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Timeout; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import software.amazon.awssdk.core.ResponseBytes; +import software.amazon.awssdk.core.async.AsyncResponseTransformer; +import software.amazon.awssdk.core.exception.SdkClientException; +import software.amazon.awssdk.services.s3.S3AsyncClient; +import software.amazon.awssdk.services.s3.model.GetObjectResponse; +import software.amazon.awssdk.services.s3.model.S3Exception; +import software.amazon.awssdk.services.s3.multipart.MultipartConfiguration; +import software.amazon.awssdk.services.s3.presignedurl.model.PresignedUrlDownloadRequest; + +@WireMockTest +@Timeout(value = 30, unit = TimeUnit.SECONDS) +class PresignedUrlMultipartDownloaderSubscriberRetryWiremockTest { + + private static final String PRESIGNED_URL_PATH = "/presigned-url"; + private static final int PART_SIZE = 16; + private static final byte[] TEST_DATA = "This is exactly a 32 byte string".getBytes(StandardCharsets.UTF_8); + private static final byte[] PART_1_DATA = Arrays.copyOfRange(TEST_DATA, 0, PART_SIZE); + private static final byte[] PART_2_DATA = Arrays.copyOfRange(TEST_DATA, PART_SIZE, TEST_DATA.length); + + private S3AsyncClient s3AsyncClient; + private URL presignedUrl; + private Path tempFile; + + @BeforeEach + void setup(WireMockRuntimeInfo wiremock) throws MalformedURLException { + s3AsyncClient = S3AsyncClient.builder() + .endpointOverride(URI.create(wiremock.getHttpBaseUrl())) + .multipartEnabled(true) + .multipartConfiguration(MultipartConfiguration.builder() + .minimumPartSizeInBytes((long) PART_SIZE) + .build()) + .build(); + presignedUrl = new URL(wiremock.getHttpBaseUrl() + PRESIGNED_URL_PATH); + } + + static Stream transformerTypes() { + return Stream.of( + Arguments.of("toBytes"), + Arguments.of("toFile") + ); + } + + @ParameterizedTest(name = "errorOnFirstPart_nonRetryable_shouldFailImmediately [{0}]") + @MethodSource("transformerTypes") + void errorOnFirstPart_nonRetryable_shouldFailImmediately(String transformerType) throws IOException { + stubError(1, 403, "AccessDeniedAccess denied"); + + assertThatThrownBy(() -> executeDownload(transformerType).join()) + .isInstanceOf(CompletionException.class) + .hasCauseInstanceOf(S3Exception.class) + .hasMessageContaining("Access denied"); + + verify(exactly(1), getRequestedFor(urlEqualTo(PRESIGNED_URL_PATH))); + } + + @ParameterizedTest(name = "errorOnMiddlePart_nonRetryable_shouldFail [{0}]") + @MethodSource("transformerTypes") + void errorOnMiddlePart_nonRetryable_shouldFail(String transformerType) throws IOException { + stubSuccessPart1(); + stubError(2, 403, "AccessDeniedAccess denied on part 2"); + + assertThatThrownBy(() -> executeDownload(transformerType).join()) + .isInstanceOf(CompletionException.class); + + verify(exactly(1), getRequestedFor(urlEqualTo(PRESIGNED_URL_PATH)) + .withHeader("Range", matching("bytes=0-15"))); + verify(exactly(1), getRequestedFor(urlEqualTo(PRESIGNED_URL_PATH)) + .withHeader("Range", matching("bytes=16-31"))); + } + + @ParameterizedTest(name = "errorOnFirstPart_retryable_exhaustsRetries_shouldFail [{0}]") + @MethodSource("transformerTypes") + void errorOnFirstPart_retryable_exhaustsRetries_shouldFail(String transformerType) throws IOException { + stubError(1, 500, "InternalErrorServer error"); + + assertThatThrownBy(() -> executeDownload(transformerType).join()) + .isInstanceOf(CompletionException.class) + .hasCauseInstanceOf(S3Exception.class) + .hasMessageContaining("Server error"); + + verify(moreThan(1), getRequestedFor(urlEqualTo(PRESIGNED_URL_PATH))); + } + + @ParameterizedTest(name = "errorOnMiddlePart_retryable_exhaustsRetries_shouldFail [{0}]") + @MethodSource("transformerTypes") + void errorOnMiddlePart_retryable_exhaustsRetries_shouldFail(String transformerType) throws IOException { + stubSuccessPart1(); + stubError(2, 500, "InternalErrorPermanent failure"); + + assertThatThrownBy(() -> executeDownload(transformerType).join()) + .isInstanceOf(CompletionException.class); + + verify(exactly(1), getRequestedFor(urlEqualTo(PRESIGNED_URL_PATH)) + .withHeader("Range", matching("bytes=0-15"))); + verify(moreThan(1), getRequestedFor(urlEqualTo(PRESIGNED_URL_PATH)) + .withHeader("Range", matching("bytes=16-31"))); + } + + @ParameterizedTest(name = "ioErrorOnFirstPart_exhaustsRetries_shouldFail [{0}]") + @MethodSource("transformerTypes") + void ioErrorOnFirstPart_exhaustsRetries_shouldFail(String transformerType) throws IOException { + stubIoError(1); + + assertThatThrownBy(() -> executeDownload(transformerType).join()) + .isInstanceOf(CompletionException.class) + .hasCauseInstanceOf(SdkClientException.class); + + verify(moreThan(1), getRequestedFor(urlEqualTo(PRESIGNED_URL_PATH)) + .withHeader("Range", matching("bytes=0-15"))); + } + + @ParameterizedTest(name = "errorOnFirstPart_retryable_thenSucceeds [{0}]") + @MethodSource("transformerTypes") + void errorOnFirstPart_retryable_thenSucceeds(String transformerType) throws IOException { + stubFor(get(urlEqualTo(PRESIGNED_URL_PATH)) + .withHeader("Range", matching("bytes=0-15")) + .inScenario("retry-first") + .whenScenarioStateIs(Scenario.STARTED) + .willReturn(aResponse() + .withStatus(500) + .withBody("InternalErrorretry")) + .willSetStateTo("retry")); + + stubFor(get(urlEqualTo(PRESIGNED_URL_PATH)) + .withHeader("Range", matching("bytes=0-15")) + .inScenario("retry-first") + .whenScenarioStateIs("retry") + .willReturn(successPart1Response()) + .willSetStateTo("part1-done")); + + stubFor(get(urlEqualTo(PRESIGNED_URL_PATH)) + .withHeader("Range", matching("bytes=16-31")) + .inScenario("retry-first") + .whenScenarioStateIs("part1-done") + .willReturn(successPart2Response())); + + Object result = executeDownload(transformerType).join(); + assertSuccessfulDownload(transformerType, result); + + verify(exactly(2), getRequestedFor(urlEqualTo(PRESIGNED_URL_PATH)) + .withHeader("Range", matching("bytes=0-15"))); + } + + @ParameterizedTest(name = "errorOnMiddlePart_retryable_thenSucceeds [{0}]") + @MethodSource("transformerTypes") + void errorOnMiddlePart_retryable_thenSucceeds(String transformerType) throws IOException { + stubSuccessPart1(); + + stubFor(get(urlEqualTo(PRESIGNED_URL_PATH)) + .withHeader("Range", matching("bytes=16-31")) + .inScenario("retry-middle") + .whenScenarioStateIs(Scenario.STARTED) + .willReturn(aResponse() + .withStatus(500) + .withBody("InternalErrorretry")) + .willSetStateTo("retry")); + + stubFor(get(urlEqualTo(PRESIGNED_URL_PATH)) + .withHeader("Range", matching("bytes=16-31")) + .inScenario("retry-middle") + .whenScenarioStateIs("retry") + .willReturn(successPart2Response())); + + Object result = executeDownload(transformerType).join(); + assertSuccessfulDownload(transformerType, result); + + verify(exactly(1), getRequestedFor(urlEqualTo(PRESIGNED_URL_PATH)) + .withHeader("Range", matching("bytes=0-15"))); + verify(exactly(2), getRequestedFor(urlEqualTo(PRESIGNED_URL_PATH)) + .withHeader("Range", matching("bytes=16-31"))); + } + + @ParameterizedTest(name = "ioErrorOnFirstPart_thenSucceeds [{0}]") + @MethodSource("transformerTypes") + void ioErrorOnFirstPart_thenSucceeds(String transformerType) throws IOException { + stubFor(get(urlEqualTo(PRESIGNED_URL_PATH)) + .withHeader("Range", matching("bytes=0-15")) + .inScenario("io-retry") + .whenScenarioStateIs(Scenario.STARTED) + .willReturn(aResponse().withFault(Fault.CONNECTION_RESET_BY_PEER)) + .willSetStateTo("retry")); + + stubFor(get(urlEqualTo(PRESIGNED_URL_PATH)) + .withHeader("Range", matching("bytes=0-15")) + .inScenario("io-retry") + .whenScenarioStateIs("retry") + .willReturn(successPart1Response()) + .willSetStateTo("part1-done")); + + stubFor(get(urlEqualTo(PRESIGNED_URL_PATH)) + .withHeader("Range", matching("bytes=16-31")) + .inScenario("io-retry") + .whenScenarioStateIs("part1-done") + .willReturn(successPart2Response())); + + Object result = executeDownload(transformerType).join(); + assertSuccessfulDownload(transformerType, result); + + verify(exactly(2), getRequestedFor(urlEqualTo(PRESIGNED_URL_PATH)) + .withHeader("Range", matching("bytes=0-15"))); + } + + @ParameterizedTest(name = "retryableError_thenUrlExpires_shouldFailWithExpiredError [{0}]") + @MethodSource("transformerTypes") + void retryableError_thenUrlExpires_shouldFailWithExpiredError(String transformerType) throws IOException { + stubFor(get(urlEqualTo(PRESIGNED_URL_PATH)) + .withHeader("Range", matching("bytes=0-15")) + .inScenario("url-expires") + .whenScenarioStateIs(Scenario.STARTED) + .willReturn(aResponse() + .withStatus(500) + .withBody("InternalErrorServer error")) + .willSetStateTo("expired")); + + stubFor(get(urlEqualTo(PRESIGNED_URL_PATH)) + .withHeader("Range", matching("bytes=0-15")) + .inScenario("url-expires") + .whenScenarioStateIs("expired") + .willReturn(aResponse() + .withStatus(403) + .withBody("AccessDenied" + + "Request has expired"))); + + assertThatThrownBy(() -> executeDownload(transformerType).join()) + .isInstanceOf(CompletionException.class) + .hasCauseInstanceOf(S3Exception.class) + .hasMessageContaining("Request has expired"); + + verify(exactly(2), getRequestedFor(urlEqualTo(PRESIGNED_URL_PATH))); + } + + + private java.util.concurrent.CompletableFuture executeDownload(String transformerType) throws IOException { + PresignedUrlDownloadRequest request = PresignedUrlDownloadRequest.builder() + .presignedUrl(presignedUrl) + .build(); + if ("toFile".equals(transformerType)) { + tempFile = Files.createTempFile("test-" + UUID.randomUUID(), ".tmp"); + Files.deleteIfExists(tempFile); + return s3AsyncClient.presignedUrlExtension().getObject(request, AsyncResponseTransformer.toFile(tempFile)); + } + return s3AsyncClient.presignedUrlExtension().getObject(request, AsyncResponseTransformer.toBytes()); + } + + @SuppressWarnings("unchecked") + private void assertSuccessfulDownload(String type, Object result) throws IOException { + if ("toBytes".equals(type)) { + assertArrayEquals(TEST_DATA, ((ResponseBytes) result).asByteArray()); + } else { + assertArrayEquals(TEST_DATA, Files.readAllBytes(tempFile)); + } + } + + private void stubSuccessPart1() { + stubFor(get(urlEqualTo(PRESIGNED_URL_PATH)) + .withHeader("Range", matching("bytes=0-15")) + .willReturn(successPart1Response())); + } + + private com.github.tomakehurst.wiremock.client.ResponseDefinitionBuilder successPart1Response() { + return aResponse() + .withStatus(206) + .withHeader("Content-Length", String.valueOf(PART_SIZE)) + .withHeader("Content-Range", "bytes 0-15/32") + .withHeader("ETag", "\"etag\"") + .withBody(PART_1_DATA); + } + + private com.github.tomakehurst.wiremock.client.ResponseDefinitionBuilder successPart2Response() { + return aResponse() + .withStatus(206) + .withHeader("Content-Length", String.valueOf(PART_SIZE)) + .withHeader("Content-Range", "bytes 16-31/32") + .withHeader("ETag", "\"etag\"") + .withBody(PART_2_DATA); + } + + private void stubError(int partNumber, int status, String body) { + String rangePattern = partNumber == 1 ? "bytes=0-15" : "bytes=16-31"; + stubFor(get(urlEqualTo(PRESIGNED_URL_PATH)) + .withHeader("Range", matching(rangePattern)) + .willReturn(aResponse() + .withStatus(status) + .withBody(body))); + } + + private void stubIoError(int partNumber) { + String rangePattern = partNumber == 1 ? "bytes=0-15" : "bytes=16-31"; + stubFor(get(urlEqualTo(PRESIGNED_URL_PATH)) + .withHeader("Range", matching(rangePattern)) + .willReturn(aResponse().withFault(Fault.CONNECTION_RESET_BY_PEER))); + } + + @AfterEach + void cleanup() { + if (s3AsyncClient != null) { + s3AsyncClient.close(); + } + if (tempFile != null && Files.exists(tempFile)) { + try { + Files.delete(tempFile); + } catch (IOException e) { + } + } + } +} diff --git a/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/multipart/PresignedUrlMultipartDownloaderSubscriberWiremockTest.java b/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/multipart/PresignedUrlMultipartDownloaderSubscriberWiremockTest.java index 90875fa524be..c71534842bb9 100644 --- a/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/multipart/PresignedUrlMultipartDownloaderSubscriberWiremockTest.java +++ b/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/multipart/PresignedUrlMultipartDownloaderSubscriberWiremockTest.java @@ -18,9 +18,9 @@ import static com.github.tomakehurst.wiremock.client.WireMock.aResponse; import static com.github.tomakehurst.wiremock.client.WireMock.absent; import static com.github.tomakehurst.wiremock.client.WireMock.get; +import static com.github.tomakehurst.wiremock.client.WireMock.matching; import static com.github.tomakehurst.wiremock.client.WireMock.stubFor; import static com.github.tomakehurst.wiremock.client.WireMock.urlEqualTo; -import static com.github.tomakehurst.wiremock.client.WireMock.matching; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.junit.jupiter.api.Assertions.assertArrayEquals; From 63d456132cdde3a7a13386820e64cbef4eff488d Mon Sep 17 00:00:00 2001 From: jencymaryjoseph <35571282+jencymaryjoseph@users.noreply.github.com> Date: Fri, 22 May 2026 14:47:21 -0700 Subject: [PATCH 20/30] Fix presigned URL download progress tracking for range requests and add listener tests (#6977) * fix progress tracking for ranged requests * Add WireMock listener tests --- ...erPresignedUrlDownloadIntegrationTest.java | 70 +++++ .../s3/internal/GenericS3TransferManager.java | 6 +- ...nagerPresignedUrlListenerWiremockTest.java | 253 ++++++++++++++++++ 3 files changed, 327 insertions(+), 2 deletions(-) create mode 100644 services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/S3TransferManagerPresignedUrlListenerWiremockTest.java diff --git a/services-custom/s3-transfer-manager/src/it/java/software/amazon/awssdk/transfer/s3/S3TransferManagerPresignedUrlDownloadIntegrationTest.java b/services-custom/s3-transfer-manager/src/it/java/software/amazon/awssdk/transfer/s3/S3TransferManagerPresignedUrlDownloadIntegrationTest.java index 0fa70b782439..9dbe520dc75d 100644 --- a/services-custom/s3-transfer-manager/src/it/java/software/amazon/awssdk/transfer/s3/S3TransferManagerPresignedUrlDownloadIntegrationTest.java +++ b/services-custom/s3-transfer-manager/src/it/java/software/amazon/awssdk/transfer/s3/S3TransferManagerPresignedUrlDownloadIntegrationTest.java @@ -41,6 +41,7 @@ import software.amazon.awssdk.testutils.RandomTempFile; import software.amazon.awssdk.transfer.s3.model.CompletedDownload; import software.amazon.awssdk.transfer.s3.model.CompletedFileDownload; +import software.amazon.awssdk.transfer.s3.model.Download; import software.amazon.awssdk.transfer.s3.model.FileDownload; import software.amazon.awssdk.transfer.s3.model.PresignedDownloadFileRequest; import software.amazon.awssdk.transfer.s3.model.PresignedDownloadRequest; @@ -109,6 +110,75 @@ void downloadWithPresignedUrl_toBytes_shouldReturnCorrectData(S3TransferManager assertThat(completed.result().asByteArray()).hasSize(objSize); } + static Stream progressTestCases() { + return Stream.of( + Arguments.of("multipart", tmJava, LARGE_KEY, null, LARGE_OBJ_SIZE), + Arguments.of("multipart", tmJava, LARGE_KEY, "bytes=0-1048575", 1048576), + Arguments.of("nonMultipart", tmNonMultipartJava, LARGE_KEY, null, LARGE_OBJ_SIZE), + Arguments.of("nonMultipart", tmNonMultipartJava, LARGE_KEY, "bytes=0-1048575", 1048576) + ); + } + + @ParameterizedTest(name = "downloadFileWithPresignedUrl_progress_{0}_range={3}") + @MethodSource("progressTestCases") + void downloadFileWithPresignedUrl_progressTracking(String tmType, S3TransferManager tm, String key, + String range, int expectedSize) throws Exception { + Path downloadPath = RandomTempFile.randomUncreatedFile().toPath(); + + PresignedUrlDownloadRequest.Builder requestBuilder = PresignedUrlDownloadRequest.builder() + .presignedUrl(createPresignedRequest(key).url()); + if (range != null) { + requestBuilder.range(range); + } + + FileDownload download = tm.downloadFileWithPresignedUrl( + PresignedDownloadFileRequest.builder() + .presignedUrlDownloadRequest(requestBuilder.build()) + .destination(downloadPath) + .addTransferListener(LoggingTransferListener.create()) + .build()); + + download.completionFuture().join(); + + // Verify progress tracking worked - totalBytes is set correctly + assertThat(download.progress().snapshot().totalBytes()).isPresent(); + assertThat(download.progress().snapshot().totalBytes().getAsLong()).isEqualTo(expectedSize); + + // Verify transferredBytes reached expectedSize + assertThat(download.progress().snapshot().transferredBytes()).isEqualTo(expectedSize); + + // Verify file size matches expected + assertThat(downloadPath.toFile().length()).isEqualTo(expectedSize); + } + + @ParameterizedTest(name = "downloadWithPresignedUrl_toBytes_progress_{0}_range={3}") + @MethodSource("progressTestCases") + void downloadWithPresignedUrl_toBytes_progressTracking(String tmType, S3TransferManager tm, String key, + String range, int expectedSize) throws Exception { + + PresignedUrlDownloadRequest.Builder requestBuilder = PresignedUrlDownloadRequest.builder() + .presignedUrl(createPresignedRequest(key).url()); + if (range != null) { + requestBuilder.range(range); + } + + Download> download = tm.downloadWithPresignedUrl( + PresignedDownloadRequest.>builder() + .presignedUrlDownloadRequest(requestBuilder.build()) + .responseTransformer(AsyncResponseTransformer.toBytes()) + .addTransferListener(LoggingTransferListener.create()) + .build()); + + CompletedDownload> completed = download.completionFuture().join(); + + assertThat(download.progress().snapshot().totalBytes()).isPresent(); + assertThat(download.progress().snapshot().totalBytes().getAsLong()).isEqualTo(expectedSize); + + assertThat(download.progress().snapshot().transferredBytes()).isEqualTo(expectedSize); + + assertThat(completed.result().asByteArray()).hasSize(expectedSize); + } + private static PresignedDownloadFileRequest createFileDownloadRequest(String key, Path destination) { return PresignedDownloadFileRequest.builder() .presignedUrlDownloadRequest(PresignedUrlDownloadRequest.builder() diff --git a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/GenericS3TransferManager.java b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/GenericS3TransferManager.java index 8d8825c67eac..8e3852a4f68a 100644 --- a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/GenericS3TransferManager.java +++ b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/GenericS3TransferManager.java @@ -613,8 +613,9 @@ public final FileDownload downloadFileWithPresignedUrl(PresignedDownloadFileRequ progressUpdater.transferInitiated(); responseTransformer = isS3ClientMultipartEnabled() + && presignedDownloadFileRequest.presignedUrlDownloadRequest().range() == null ? progressUpdater.wrapForNonSerialFileDownload( - responseTransformer, GetObjectRequest.builder().build()) + responseTransformer, GetObjectRequest.builder().build()) : progressUpdater.wrapResponseTransformer(responseTransformer); progressUpdater.registerCompletion(returnFuture); @@ -652,8 +653,9 @@ public final Download downloadWithPresignedUrl( progressUpdater.transferInitiated(); responseTransformer = isS3ClientMultipartEnabled() + && presignedDownloadRequest.presignedUrlDownloadRequest().range() == null ? progressUpdater.wrapForNonSerialFileDownload( - responseTransformer, GetObjectRequest.builder().build()) + responseTransformer, GetObjectRequest.builder().build()) : progressUpdater.wrapResponseTransformer(responseTransformer); progressUpdater.registerCompletion(returnFuture); diff --git a/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/S3TransferManagerPresignedUrlListenerWiremockTest.java b/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/S3TransferManagerPresignedUrlListenerWiremockTest.java new file mode 100644 index 000000000000..8a1ee54582b3 --- /dev/null +++ b/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/S3TransferManagerPresignedUrlListenerWiremockTest.java @@ -0,0 +1,253 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.transfer.s3.internal; + +import static com.github.tomakehurst.wiremock.client.WireMock.aResponse; +import static com.github.tomakehurst.wiremock.client.WireMock.get; +import static com.github.tomakehurst.wiremock.client.WireMock.stubFor; +import static com.github.tomakehurst.wiremock.client.WireMock.urlPathEqualTo; +import static org.assertj.core.api.Assertions.assertThatExceptionOfType; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.timeout; +import static org.mockito.Mockito.times; + +import com.github.tomakehurst.wiremock.client.WireMock; +import com.github.tomakehurst.wiremock.junit5.WireMockRuntimeInfo; +import com.github.tomakehurst.wiremock.junit5.WireMockTest; +import java.io.IOException; +import java.net.URI; +import java.net.URL; +import java.util.concurrent.CompletionException; +import java.util.stream.Stream; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import org.mockito.ArgumentMatchers; +import org.mockito.Mockito; +import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; +import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; +import software.amazon.awssdk.core.async.AsyncResponseTransformer; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.s3.S3AsyncClient; +import software.amazon.awssdk.services.s3.presignedurl.model.PresignedUrlDownloadRequest; +import software.amazon.awssdk.testutils.RandomTempFile; +import software.amazon.awssdk.transfer.s3.S3TransferManager; +import software.amazon.awssdk.transfer.s3.model.Download; +import software.amazon.awssdk.transfer.s3.model.FileDownload; +import software.amazon.awssdk.transfer.s3.model.PresignedDownloadFileRequest; +import software.amazon.awssdk.transfer.s3.model.PresignedDownloadRequest; +import software.amazon.awssdk.transfer.s3.progress.TransferListener; + +/** + * Tests that TransferListener callbacks fire correctly for presigned URL downloads + * with both multipart-enabled and non-multipart clients. + */ +@WireMockTest +public class S3TransferManagerPresignedUrlListenerWiremockTest { + + private static URI testEndpoint; + private static RandomTempFile testFile; + + @BeforeAll + public static void init(WireMockRuntimeInfo wm) throws IOException { + testEndpoint = URI.create(wm.getHttpBaseUrl()); + testFile = new RandomTempFile("presigned-listener-test", 1024); + } + + @BeforeEach + void resetWireMock() { + WireMock.reset(); + } + + private static S3AsyncClient s3AsyncClient(boolean multipartEnabled) { + return S3AsyncClient.builder() + .multipartEnabled(multipartEnabled) + .region(Region.US_EAST_1) + .endpointOverride(testEndpoint) + .credentialsProvider( + StaticCredentialsProvider.create(AwsBasicCredentials.create("key", "secret"))) + .build(); + } + + static Stream presignedUrlTestCases() { + return Stream.of( + Arguments.of(true, "toFile", null), + Arguments.of(true, "toFile", "bytes=0-511"), + Arguments.of(true, "toBytes", null), + Arguments.of(true, "toBytes", "bytes=0-511"), + Arguments.of(false, "toFile", null), + Arguments.of(false, "toFile", "bytes=0-511"), + Arguments.of(false, "toBytes", null), + Arguments.of(false, "toBytes", "bytes=0-511") + ); + } + + @ParameterizedTest(name = "presignedUrlDownload_multipart={0}_type={1}_range={2}") + @MethodSource("presignedUrlTestCases") + void presignedUrlDownload_shouldInvokeListener(boolean multipartEnabled, String type, String range) throws Exception { + S3AsyncClient s3Async = s3AsyncClient(multipartEnabled); + S3TransferManager tm = new GenericS3TransferManager(s3Async, mock(UploadDirectoryHelper.class), + mock(TransferManagerConfiguration.class), + mock(DownloadDirectoryHelper.class)); + + byte[] responseBody = new byte[512]; + stubFor(get(urlPathEqualTo("/presigned-key")).willReturn(aResponse() + .withStatus(206) + .withHeader("Content-Length", "512") + .withHeader("Content-Range", "bytes 0-511/512") + .withHeader("ETag", "\"test-etag\"") + .withBody(responseBody))); + + TransferListener listener = mock(TransferListener.class); + URL presignedUrl = new URL(testEndpoint + "/presigned-key?X-Amz-Algorithm=AWS4-HMAC-SHA256"); + + PresignedUrlDownloadRequest.Builder requestBuilder = PresignedUrlDownloadRequest.builder() + .presignedUrl(presignedUrl); + if (range != null) { + requestBuilder.range(range); + } + + if ("toFile".equals(type)) { + FileDownload download = tm.downloadFileWithPresignedUrl( + PresignedDownloadFileRequest.builder() + .presignedUrlDownloadRequest(requestBuilder.build()) + .destination(testFile.toPath()) + .addTransferListener(listener) + .build()); + download.completionFuture().join(); + } else { + Download download = tm.downloadWithPresignedUrl( + PresignedDownloadRequest.builder() + .presignedUrlDownloadRequest(requestBuilder.build()) + .responseTransformer(AsyncResponseTransformer.toBytes()) + .addTransferListener(listener) + .build()); + download.completionFuture().join(); + } + + Mockito.verify(listener, timeout(1000).times(1)).transferInitiated(ArgumentMatchers.any()); + Mockito.verify(listener, timeout(1000).atLeastOnce()).bytesTransferred(ArgumentMatchers.any()); + + tm.close(); + s3Async.close(); + } + + static Stream presignedUrlFailureTestCases() { + return Stream.of( + Arguments.of(true, "toFile"), + Arguments.of(true, "toBytes"), + Arguments.of(false, "toFile"), + Arguments.of(false, "toBytes") + ); + } + + @ParameterizedTest(name = "presignedUrlDownload_failure_multipart={0}_type={1}") + @MethodSource("presignedUrlFailureTestCases") + void presignedUrlDownload_failure_shouldInvokeListener(boolean multipartEnabled, String type) throws Exception { + S3AsyncClient s3Async = s3AsyncClient(multipartEnabled); + S3TransferManager tm = new GenericS3TransferManager(s3Async, mock(UploadDirectoryHelper.class), + mock(TransferManagerConfiguration.class), + mock(DownloadDirectoryHelper.class)); + + stubFor(get(urlPathEqualTo("/presigned-key")) + .willReturn(aResponse().withStatus(404) + .withBody("TestErrorTest failure"))); + + TransferListener listener = mock(TransferListener.class); + URL presignedUrl = new URL(testEndpoint + "/presigned-key?X-Amz-Algorithm=AWS4-HMAC-SHA256"); + + if ("toFile".equals(type)) { + FileDownload download = tm.downloadFileWithPresignedUrl( + PresignedDownloadFileRequest.builder() + .presignedUrlDownloadRequest(PresignedUrlDownloadRequest.builder() + .presignedUrl(presignedUrl) + .build()) + .destination(testFile.toPath()) + .addTransferListener(listener) + .build()); + assertThatExceptionOfType(CompletionException.class).isThrownBy(() -> download.completionFuture().join()); + } else { + Download download = tm.downloadWithPresignedUrl( + PresignedDownloadRequest.builder() + .presignedUrlDownloadRequest(PresignedUrlDownloadRequest.builder() + .presignedUrl(presignedUrl) + .build()) + .responseTransformer(AsyncResponseTransformer.toBytes()) + .addTransferListener(listener) + .build()); + assertThatExceptionOfType(CompletionException.class).isThrownBy(() -> download.completionFuture().join()); + } + + Mockito.verify(listener, timeout(1000).times(1)).transferInitiated(ArgumentMatchers.any()); + Mockito.verify(listener, timeout(1000).times(1)).transferFailed(ArgumentMatchers.any()); + Mockito.verify(listener, times(0)).transferComplete(ArgumentMatchers.any()); + + tm.close(); + s3Async.close(); + } + + @ParameterizedTest(name = "presignedUrlDownload_cancelled_multipart={0}_type={1}") + @MethodSource("presignedUrlFailureTestCases") + void presignedUrlDownload_cancelled_shouldInvokeTransferFailed(boolean multipartEnabled, String type) throws Exception { + S3AsyncClient s3Async = s3AsyncClient(multipartEnabled); + S3TransferManager tm = new GenericS3TransferManager(s3Async, mock(UploadDirectoryHelper.class), + mock(TransferManagerConfiguration.class), + mock(DownloadDirectoryHelper.class)); + + // Slow response to keep request in-flight during cancellation + stubFor(get(urlPathEqualTo("/presigned-key")).willReturn(aResponse() + .withStatus(206) + .withHeader("Content-Length", "512") + .withHeader("Content-Range", "bytes 0-511/512") + .withHeader("ETag", "\"test-etag\"") + .withBody(new byte[512]) + .withFixedDelay(5000))); + + TransferListener listener = mock(TransferListener.class); + URL presignedUrl = new URL(testEndpoint + "/presigned-key?X-Amz-Algorithm=AWS4-HMAC-SHA256"); + + if ("toFile".equals(type)) { + FileDownload download = tm.downloadFileWithPresignedUrl( + PresignedDownloadFileRequest.builder() + .presignedUrlDownloadRequest(PresignedUrlDownloadRequest.builder() + .presignedUrl(presignedUrl) + .build()) + .destination(testFile.toPath()) + .addTransferListener(listener) + .build()); + download.completionFuture().cancel(true); + } else { + Download download = tm.downloadWithPresignedUrl( + PresignedDownloadRequest.builder() + .presignedUrlDownloadRequest(PresignedUrlDownloadRequest.builder() + .presignedUrl(presignedUrl) + .build()) + .responseTransformer(AsyncResponseTransformer.toBytes()) + .addTransferListener(listener) + .build()); + download.completionFuture().cancel(true); + } + + Mockito.verify(listener, timeout(1000).times(1)).transferInitiated(ArgumentMatchers.any()); + Mockito.verify(listener, timeout(1000).times(1)).transferFailed(ArgumentMatchers.any()); + Mockito.verify(listener, times(0)).transferComplete(ArgumentMatchers.any()); + + tm.close(); + s3Async.close(); + } +} From e126cb2f72aeb41d9b84f47fb9c54bf5964533f8 Mon Sep 17 00:00:00 2001 From: jencymaryjoseph <35571282+jencymaryjoseph@users.noreply.github.com> Date: Mon, 1 Jun 2026 12:12:18 -0700 Subject: [PATCH 21/30] Add PresignedFileDownload return type for presigned URL downloads (#7004) * Add PresignedFileDownload return type for presigned URL downloads * Address PR Comments --- ...erPresignedUrlDownloadIntegrationTest.java | 6 +- .../awssdk/transfer/s3/S3TransferManager.java | 9 ++- .../internal/DelegatingS3TransferManager.java | 3 +- .../s3/internal/GenericS3TransferManager.java | 10 +-- .../model/DefaultPresignedFileDownload.java | 79 +++++++++++++++++++ .../s3/model/PresignedFileDownload.java | 33 ++++++++ .../DefaultPresignedFileDownloadTest.java | 30 +++++++ ...ansferManagerPresignedUrlDownloadTest.java | 13 --- ...nagerPresignedUrlListenerWiremockTest.java | 8 +- 9 files changed, 160 insertions(+), 31 deletions(-) create mode 100644 services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/model/DefaultPresignedFileDownload.java create mode 100644 services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/model/PresignedFileDownload.java create mode 100644 services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/DefaultPresignedFileDownloadTest.java diff --git a/services-custom/s3-transfer-manager/src/it/java/software/amazon/awssdk/transfer/s3/S3TransferManagerPresignedUrlDownloadIntegrationTest.java b/services-custom/s3-transfer-manager/src/it/java/software/amazon/awssdk/transfer/s3/S3TransferManagerPresignedUrlDownloadIntegrationTest.java index 9dbe520dc75d..003fddb5d5b7 100644 --- a/services-custom/s3-transfer-manager/src/it/java/software/amazon/awssdk/transfer/s3/S3TransferManagerPresignedUrlDownloadIntegrationTest.java +++ b/services-custom/s3-transfer-manager/src/it/java/software/amazon/awssdk/transfer/s3/S3TransferManagerPresignedUrlDownloadIntegrationTest.java @@ -41,8 +41,8 @@ import software.amazon.awssdk.testutils.RandomTempFile; import software.amazon.awssdk.transfer.s3.model.CompletedDownload; import software.amazon.awssdk.transfer.s3.model.CompletedFileDownload; +import software.amazon.awssdk.transfer.s3.model.PresignedFileDownload; import software.amazon.awssdk.transfer.s3.model.Download; -import software.amazon.awssdk.transfer.s3.model.FileDownload; import software.amazon.awssdk.transfer.s3.model.PresignedDownloadFileRequest; import software.amazon.awssdk.transfer.s3.model.PresignedDownloadRequest; import software.amazon.awssdk.transfer.s3.progress.LoggingTransferListener; @@ -92,7 +92,7 @@ static Stream testCases() { void downloadFileWithPresignedUrl_shouldDownloadCorrectly(S3TransferManager tm, String key, File sourceFile, int objSize) throws Exception { Path downloadPath = RandomTempFile.randomUncreatedFile().toPath(); - FileDownload download = tm.downloadFileWithPresignedUrl(createFileDownloadRequest(key, downloadPath)); + PresignedFileDownload download = tm.downloadFileWithPresignedUrl(createFileDownloadRequest(key, downloadPath)); CompletedFileDownload completed = download.completionFuture().join(); assertThat(Files.exists(downloadPath)).isTrue(); @@ -131,7 +131,7 @@ void downloadFileWithPresignedUrl_progressTracking(String tmType, S3TransferMana requestBuilder.range(range); } - FileDownload download = tm.downloadFileWithPresignedUrl( + PresignedFileDownload download = tm.downloadFileWithPresignedUrl( PresignedDownloadFileRequest.builder() .presignedUrlDownloadRequest(requestBuilder.build()) .destination(downloadPath) diff --git a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/S3TransferManager.java b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/S3TransferManager.java index 0f54a432a76b..aaa5916f7b1b 100644 --- a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/S3TransferManager.java +++ b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/S3TransferManager.java @@ -43,6 +43,7 @@ import software.amazon.awssdk.transfer.s3.model.FileUpload; import software.amazon.awssdk.transfer.s3.model.PresignedDownloadFileRequest; import software.amazon.awssdk.transfer.s3.model.PresignedDownloadRequest; +import software.amazon.awssdk.transfer.s3.model.PresignedFileDownload; import software.amazon.awssdk.transfer.s3.model.ResumableFileDownload; import software.amazon.awssdk.transfer.s3.model.ResumableFileUpload; import software.amazon.awssdk.transfer.s3.model.Upload; @@ -730,16 +731,16 @@ default Copy copy(Consumer copyRequestBuilder) { * LoggingTransferListener.create()) * .build(); * - * FileDownload download = transferManager.downloadFileWithPresignedUrl(request); + * PresignedFileDownload download = transferManager.downloadFileWithPresignedUrl(request); * download.completionFuture().join(); * } * * @param presignedDownloadFileRequest the presigned download file request - * @return A {@link FileDownload} that can be used to track the ongoing transfer + * @return A {@link PresignedFileDownload} that can be used to track the ongoing transfer * @see #downloadFileWithPresignedUrl(Consumer) * @see #downloadWithPresignedUrl(PresignedDownloadRequest) */ - default FileDownload downloadFileWithPresignedUrl(PresignedDownloadFileRequest presignedDownloadFileRequest) { + default PresignedFileDownload downloadFileWithPresignedUrl(PresignedDownloadFileRequest presignedDownloadFileRequest) { throw new UnsupportedOperationException(); } @@ -752,7 +753,7 @@ default FileDownload downloadFileWithPresignedUrl(PresignedDownloadFileRequest p * * @see #downloadFileWithPresignedUrl(PresignedDownloadFileRequest) */ - default FileDownload downloadFileWithPresignedUrl( + default PresignedFileDownload downloadFileWithPresignedUrl( Consumer presignedDownloadFileRequest) { return downloadFileWithPresignedUrl( PresignedDownloadFileRequest.builder().applyMutation(presignedDownloadFileRequest).build()); diff --git a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/DelegatingS3TransferManager.java b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/DelegatingS3TransferManager.java index 15d309552857..6907f1f00d08 100644 --- a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/DelegatingS3TransferManager.java +++ b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/DelegatingS3TransferManager.java @@ -29,6 +29,7 @@ import software.amazon.awssdk.transfer.s3.model.FileUpload; import software.amazon.awssdk.transfer.s3.model.PresignedDownloadFileRequest; import software.amazon.awssdk.transfer.s3.model.PresignedDownloadRequest; +import software.amazon.awssdk.transfer.s3.model.PresignedFileDownload; import software.amazon.awssdk.transfer.s3.model.ResumableFileDownload; import software.amazon.awssdk.transfer.s3.model.Upload; import software.amazon.awssdk.transfer.s3.model.UploadDirectoryRequest; @@ -88,7 +89,7 @@ public Copy copy(CopyRequest copyRequest) { } @Override - public FileDownload downloadFileWithPresignedUrl(PresignedDownloadFileRequest presignedDownloadFileRequest) { + public PresignedFileDownload downloadFileWithPresignedUrl(PresignedDownloadFileRequest presignedDownloadFileRequest) { return delegate.downloadFileWithPresignedUrl(presignedDownloadFileRequest); } diff --git a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/GenericS3TransferManager.java b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/GenericS3TransferManager.java index 8e3852a4f68a..c86a3fea12fb 100644 --- a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/GenericS3TransferManager.java +++ b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/GenericS3TransferManager.java @@ -55,6 +55,7 @@ import software.amazon.awssdk.transfer.s3.internal.model.DefaultDownload; import software.amazon.awssdk.transfer.s3.internal.model.DefaultFileDownload; import software.amazon.awssdk.transfer.s3.internal.model.DefaultFileUpload; +import software.amazon.awssdk.transfer.s3.internal.model.DefaultPresignedFileDownload; import software.amazon.awssdk.transfer.s3.internal.model.DefaultUpload; import software.amazon.awssdk.transfer.s3.internal.progress.DefaultTransferProgress; import software.amazon.awssdk.transfer.s3.internal.progress.DefaultTransferProgressSnapshot; @@ -77,6 +78,7 @@ import software.amazon.awssdk.transfer.s3.model.FileUpload; import software.amazon.awssdk.transfer.s3.model.PresignedDownloadFileRequest; import software.amazon.awssdk.transfer.s3.model.PresignedDownloadRequest; +import software.amazon.awssdk.transfer.s3.model.PresignedFileDownload; import software.amazon.awssdk.transfer.s3.model.ResumableFileDownload; import software.amazon.awssdk.transfer.s3.model.ResumableFileUpload; import software.amazon.awssdk.transfer.s3.model.Upload; @@ -600,7 +602,7 @@ public final Copy copy(CopyRequest copyRequest) { } @Override - public final FileDownload downloadFileWithPresignedUrl(PresignedDownloadFileRequest presignedDownloadFileRequest) { + public final PresignedFileDownload downloadFileWithPresignedUrl(PresignedDownloadFileRequest presignedDownloadFileRequest) { Validate.paramNotNull(presignedDownloadFileRequest, "presignedDownloadFileRequest"); AsyncResponseTransformer responseTransformer = @@ -632,11 +634,7 @@ public final FileDownload downloadFileWithPresignedUrl(PresignedDownloadFileRequ returnFuture.completeExceptionally(throwable); } - return new DefaultFileDownload(returnFuture, progressUpdater.progress(), - () -> { - throw new UnsupportedOperationException( - "Pause is not supported for presigned URL downloads"); - }, null); + return new DefaultPresignedFileDownload(returnFuture, progressUpdater.progress()); } @Override diff --git a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/model/DefaultPresignedFileDownload.java b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/model/DefaultPresignedFileDownload.java new file mode 100644 index 000000000000..253c60a8a42a --- /dev/null +++ b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/model/DefaultPresignedFileDownload.java @@ -0,0 +1,79 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.transfer.s3.internal.model; + +import java.util.Objects; +import java.util.concurrent.CompletableFuture; +import software.amazon.awssdk.annotations.SdkInternalApi; +import software.amazon.awssdk.transfer.s3.model.CompletedFileDownload; +import software.amazon.awssdk.transfer.s3.model.PresignedFileDownload; +import software.amazon.awssdk.transfer.s3.progress.TransferProgress; +import software.amazon.awssdk.utils.ToString; +import software.amazon.awssdk.utils.Validate; + +@SdkInternalApi +public final class DefaultPresignedFileDownload implements PresignedFileDownload { + private final CompletableFuture completionFuture; + private final TransferProgress progress; + + public DefaultPresignedFileDownload(CompletableFuture completionFuture, + TransferProgress progress) { + this.completionFuture = Validate.paramNotNull(completionFuture, "completionFuture"); + this.progress = Validate.paramNotNull(progress, "progress"); + } + + @Override + public CompletableFuture completionFuture() { + return completionFuture; + } + + @Override + public TransferProgress progress() { + return progress; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + + DefaultPresignedFileDownload that = (DefaultPresignedFileDownload) o; + + if (!Objects.equals(completionFuture, that.completionFuture)) { + return false; + } + return Objects.equals(progress, that.progress); + } + + @Override + public int hashCode() { + int result = completionFuture != null ? completionFuture.hashCode() : 0; + result = 31 * result + (progress != null ? progress.hashCode() : 0); + return result; + } + + @Override + public String toString() { + return ToString.builder("DefaultPresignedFileDownload") + .add("completionFuture", completionFuture) + .add("progress", progress) + .build(); + } +} diff --git a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/model/PresignedFileDownload.java b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/model/PresignedFileDownload.java new file mode 100644 index 000000000000..669712c2dea1 --- /dev/null +++ b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/model/PresignedFileDownload.java @@ -0,0 +1,33 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.transfer.s3.model; + +import java.util.concurrent.CompletableFuture; +import software.amazon.awssdk.annotations.SdkPublicApi; +import software.amazon.awssdk.annotations.ThreadSafe; + +/** + * A download transfer of a single object from S3 using a presigned URL. + * + *

Unlike {@link FileDownload}, this type does not support pause/resume. + */ +@SdkPublicApi +@ThreadSafe +public interface PresignedFileDownload extends ObjectTransfer { + + @Override + CompletableFuture completionFuture(); +} diff --git a/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/DefaultPresignedFileDownloadTest.java b/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/DefaultPresignedFileDownloadTest.java new file mode 100644 index 000000000000..b98a4cb35454 --- /dev/null +++ b/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/DefaultPresignedFileDownloadTest.java @@ -0,0 +1,30 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.transfer.s3.internal; + +import nl.jqno.equalsverifier.EqualsVerifier; +import org.junit.jupiter.api.Test; +import software.amazon.awssdk.transfer.s3.internal.model.DefaultPresignedFileDownload; + +public class DefaultPresignedFileDownloadTest { + + @Test + public void equals_hashcode() { + EqualsVerifier.forClass(DefaultPresignedFileDownload.class) + .withNonnullFields("completionFuture", "progress") + .verify(); + } +} diff --git a/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/S3TransferManagerPresignedUrlDownloadTest.java b/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/S3TransferManagerPresignedUrlDownloadTest.java index 16de24bec605..a299e5ca41f7 100644 --- a/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/S3TransferManagerPresignedUrlDownloadTest.java +++ b/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/S3TransferManagerPresignedUrlDownloadTest.java @@ -35,7 +35,6 @@ import software.amazon.awssdk.services.s3.presignedurl.model.PresignedUrlDownloadRequest; import software.amazon.awssdk.transfer.s3.model.CompletedDownload; import software.amazon.awssdk.transfer.s3.model.CompletedFileDownload; -import software.amazon.awssdk.transfer.s3.model.FileDownload; import software.amazon.awssdk.transfer.s3.model.PresignedDownloadFileRequest; import software.amazon.awssdk.transfer.s3.model.PresignedDownloadRequest; @@ -154,18 +153,6 @@ void downloadWithPresignedUrl_withNullRequest_shouldThrowNullPointerException() assertThatThrownBy(() -> tm.downloadWithPresignedUrl(null)) .isInstanceOf(NullPointerException.class); } - @Test - void downloadFileWithPresignedUrl_pause_shouldThrowUnsupportedOperationException() { - GetObjectResponse response = GetObjectResponse.builder().build(); - stubGetObject(CompletableFuture.completedFuture(response)); - - FileDownload download = tm.downloadFileWithPresignedUrl(fileDownloadRequest()); - download.completionFuture().join(); - - assertThatThrownBy(download::pause) - .isInstanceOf(UnsupportedOperationException.class) - .hasMessageContaining("Pause is not supported for presigned URL downloads"); - } private PresignedDownloadFileRequest fileDownloadRequest() { return PresignedDownloadFileRequest.builder() diff --git a/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/S3TransferManagerPresignedUrlListenerWiremockTest.java b/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/S3TransferManagerPresignedUrlListenerWiremockTest.java index 8a1ee54582b3..2e9e5dbecca8 100644 --- a/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/S3TransferManagerPresignedUrlListenerWiremockTest.java +++ b/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/S3TransferManagerPresignedUrlListenerWiremockTest.java @@ -48,7 +48,7 @@ import software.amazon.awssdk.testutils.RandomTempFile; import software.amazon.awssdk.transfer.s3.S3TransferManager; import software.amazon.awssdk.transfer.s3.model.Download; -import software.amazon.awssdk.transfer.s3.model.FileDownload; +import software.amazon.awssdk.transfer.s3.model.PresignedFileDownload; import software.amazon.awssdk.transfer.s3.model.PresignedDownloadFileRequest; import software.amazon.awssdk.transfer.s3.model.PresignedDownloadRequest; import software.amazon.awssdk.transfer.s3.progress.TransferListener; @@ -123,7 +123,7 @@ void presignedUrlDownload_shouldInvokeListener(boolean multipartEnabled, String } if ("toFile".equals(type)) { - FileDownload download = tm.downloadFileWithPresignedUrl( + PresignedFileDownload download = tm.downloadFileWithPresignedUrl( PresignedDownloadFileRequest.builder() .presignedUrlDownloadRequest(requestBuilder.build()) .destination(testFile.toPath()) @@ -172,7 +172,7 @@ void presignedUrlDownload_failure_shouldInvokeListener(boolean multipartEnabled, URL presignedUrl = new URL(testEndpoint + "/presigned-key?X-Amz-Algorithm=AWS4-HMAC-SHA256"); if ("toFile".equals(type)) { - FileDownload download = tm.downloadFileWithPresignedUrl( + PresignedFileDownload download = tm.downloadFileWithPresignedUrl( PresignedDownloadFileRequest.builder() .presignedUrlDownloadRequest(PresignedUrlDownloadRequest.builder() .presignedUrl(presignedUrl) @@ -222,7 +222,7 @@ void presignedUrlDownload_cancelled_shouldInvokeTransferFailed(boolean multipart URL presignedUrl = new URL(testEndpoint + "/presigned-key?X-Amz-Algorithm=AWS4-HMAC-SHA256"); if ("toFile".equals(type)) { - FileDownload download = tm.downloadFileWithPresignedUrl( + PresignedFileDownload download = tm.downloadFileWithPresignedUrl( PresignedDownloadFileRequest.builder() .presignedUrlDownloadRequest(PresignedUrlDownloadRequest.builder() .presignedUrl(presignedUrl) From ba66c7b51bfbd7c1d9a3257078fad5c32daeb401 Mon Sep 17 00:00:00 2001 From: jencymaryjoseph <35571282+jencymaryjoseph@users.noreply.github.com> Date: Tue, 9 Jun 2026 15:03:00 -0700 Subject: [PATCH 22/30] Fix first part response validation in parallel presigned URL multipart subscriber (#7013) * Fix first part validation and refactor * Use String.format for exceptions --- .../multipart/MultipartDownloadUtils.java | 28 ++- ...ignedUrlMultipartDownloaderSubscriber.java | 75 +++----- .../multipart/PresignedUrlDownloadHelper.java | 88 +++++++++ ...ignedUrlMultipartDownloaderSubscriber.java | 102 ++-------- .../multipart/MultipartDownloadUtilsTest.java | 25 ++- ...dUrlMultipartDownloaderSubscriberTest.java | 78 ++++++++ .../PresignedUrlDownloadHelperTest.java | 182 ++++++++++++++++++ 7 files changed, 432 insertions(+), 146 deletions(-) create mode 100644 services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/multipart/PresignedUrlDownloadHelperTest.java diff --git a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/MultipartDownloadUtils.java b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/MultipartDownloadUtils.java index 8be5bb8ee5b4..cf1568b7a432 100644 --- a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/MultipartDownloadUtils.java +++ b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/MultipartDownloadUtils.java @@ -78,20 +78,23 @@ public static Optional multipartDownloadResumeCo } /** - * Parses the start byte from a Content-Range header. - * + * Parses start byte and end byte from a Content-Range header. + * * @param contentRange the Content-Range header value (e.g., "bytes 0-1023/2048") - * @return the start byte position, or -1 if parsing fails + * @return array of [startByte, endByte], or null if parsing fails */ - public static long parseStartByteFromContentRange(String contentRange) { + public static long[] parseContentRange(String contentRange) { if (contentRange == null) { - return -1; + return null; } Matcher matcher = CONTENT_RANGE_PATTERN.matcher(contentRange); if (!matcher.matches()) { - return -1; + return null; } - return Long.parseLong(matcher.group(1)); + return new long[] { + Long.parseLong(matcher.group(1)), + Long.parseLong(matcher.group(2)) + }; } /** @@ -111,4 +114,15 @@ public static Optional parseContentRangeForTotalSize(String contentRange) return Optional.of(Long.parseLong(matcher.group(3))); } + /** + * Calculates the total number of parts needed to download an object of the given size. + * + * @param contentLength total object size in bytes + * @param partSize size of each part in bytes + * @return the number of parts + */ + public static int calculateTotalParts(long contentLength, long partSize) { + return (int) Math.ceil((double) contentLength / partSize); + } + } diff --git a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/ParallelPresignedUrlMultipartDownloaderSubscriber.java b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/ParallelPresignedUrlMultipartDownloaderSubscriber.java index 8df801f96d34..5c62778d85ee 100644 --- a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/ParallelPresignedUrlMultipartDownloaderSubscriber.java +++ b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/ParallelPresignedUrlMultipartDownloaderSubscriber.java @@ -50,7 +50,6 @@ public class ParallelPresignedUrlMultipartDownloaderSubscriber implements Subscriber> { private static final Logger log = Logger.loggerFor(ParallelPresignedUrlMultipartDownloaderSubscriber.class); - private static final String BYTES_RANGE_PREFIX = "bytes="; private final S3AsyncClient s3AsyncClient; private final PresignedUrlDownloadRequest presignedUrlDownloadRequest; @@ -66,7 +65,10 @@ public class ParallelPresignedUrlMultipartDownloaderSubscriber private final AtomicInteger partNumber = new AtomicInteger(0); private final AtomicInteger completedParts = new AtomicInteger(0); private final Semaphore inFlightPermits; - private final AtomicBoolean isCompletedExceptionally = new AtomicBoolean(false); + /** + * CAS gate ensuring only the first part failure triggers error handling and cancellation. + */ + private final AtomicBoolean downloadFailed = new AtomicBoolean(false); private final AtomicBoolean processingPending = new AtomicBoolean(false); private final Map> inFlightRequests = new ConcurrentHashMap<>(); private final Queue>> pendingTransformers = @@ -138,6 +140,7 @@ private void sendFirstRequest(AsyncResponseTransformer { inFlightRequests.remove(0); @@ -156,8 +159,7 @@ private void sendFirstRequest(AsyncResponseTransformer String.format("Total content length: %d, Total parts: %d", totalContentLength, totalParts)); + Optional validationError = validatePartResponse(res, 0); + if (validationError.isPresent()) { + handlePartError(validationError.get(), 0); + return; + } + if (totalParts <= 1) { resultFuture.complete(firstResponse); synchronized (subscriptionLock) { @@ -217,7 +225,7 @@ private void processRequest(AsyncResponseTransformer transformer, int partIndex) { - if (isCompletedExceptionally.get()) { + if (downloadFailed.get()) { inFlightPermits.release(); return; } @@ -235,10 +243,14 @@ private void sendPartRequest(AsyncResponseTransformer "Ignoring late completion for part " + partIndex + ", download already failed"); + return; + } Optional validationError = validatePartResponse(res, partIndex); if (validationError.isPresent()) { @@ -288,35 +300,12 @@ private void processPendingTransformers() { } private Optional validatePartResponse(GetObjectResponse response, int partIndex) { - String contentRange = response.contentRange(); - if (contentRange == null) { - return Optional.of(PresignedUrlDownloadHelper.missingContentRangeHeader()); - } - Long contentLength = response.contentLength(); - if (contentLength == null || contentLength < 0) { - return Optional.of(PresignedUrlDownloadHelper.invalidContentLength()); - } - long expectedStartByte = partIndex * configuredPartSizeInBytes; - long expectedEndByte = Math.min(expectedStartByte + configuredPartSizeInBytes - 1, totalContentLength - 1); - String expectedRange = "bytes " + expectedStartByte + "-" + expectedEndByte + "/"; - if (!contentRange.startsWith(expectedRange)) { - return Optional.of(SdkClientException.create( - "Content-Range mismatch. Expected range starting with: " + expectedRange + - ", but got: " + contentRange)); - } - long expectedPartSize = (partIndex == totalParts - 1) - ? totalContentLength - (partIndex * configuredPartSizeInBytes) - : configuredPartSizeInBytes; - if (!contentLength.equals(expectedPartSize)) { - return Optional.of(SdkClientException.create( - String.format("Part content length validation failed for part %d. Expected: %d, but got: %d", - partIndex, expectedPartSize, contentLength))); - } - return Optional.empty(); + return PresignedUrlDownloadHelper.validatePartResponse( + response, partIndex, configuredPartSizeInBytes, totalContentLength, totalParts); } private void handlePartError(Throwable error, int partIndex) { - if (isCompletedExceptionally.compareAndSet(false, true)) { + if (downloadFailed.compareAndSet(false, true)) { log.debug(() -> "Error on part " + partIndex, error); resultFuture.completeExceptionally(error); inFlightRequests.values().forEach(future -> future.cancel(true)); @@ -329,24 +318,8 @@ private void handlePartError(Throwable error, int partIndex) { } private PresignedUrlDownloadRequest createRangedGetRequest(int partIndex) { - long startByte = partIndex * configuredPartSizeInBytes; - long endByte; - if (totalContentLength != null) { - endByte = Math.min(startByte + configuredPartSizeInBytes - 1, totalContentLength - 1); - } else { - endByte = startByte + configuredPartSizeInBytes - 1; - } - String rangeHeader = BYTES_RANGE_PREFIX + startByte + "-" + endByte; - PresignedUrlDownloadRequest.Builder builder = presignedUrlDownloadRequest.toBuilder() - .range(rangeHeader); - if (partIndex > 0 && eTag != null) { - builder.ifMatch(eTag); - } - return builder.build(); - } - - private int calculateTotalParts(long contentLength, long partSize) { - return (int) Math.ceil((double) contentLength / partSize); + return PresignedUrlDownloadHelper.createRangedGetRequest( + presignedUrlDownloadRequest, partIndex, configuredPartSizeInBytes, totalContentLength, eTag); } @Override diff --git a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/PresignedUrlDownloadHelper.java b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/PresignedUrlDownloadHelper.java index 3f5c955abad7..9cc521fd98f2 100644 --- a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/PresignedUrlDownloadHelper.java +++ b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/PresignedUrlDownloadHelper.java @@ -15,6 +15,7 @@ package software.amazon.awssdk.services.s3.internal.multipart; +import java.util.Optional; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionException; import software.amazon.awssdk.annotations.SdkInternalApi; @@ -142,6 +143,93 @@ static SdkClientException invalidContentLength() { return SdkClientException.create("Invalid or missing Content-Length in response"); } + /** + * Validates a part response for data integrity. Checks that Content-Range and Content-Length + * match the expected values based on part index, part size, and total object size. + * + * @param response the GetObject response to validate + * @param partIndex zero-based index of this part + * @param partSizeInBytes configured part size + * @param totalContentLength total object size (from Content-Range), or null if not yet known + * @param totalParts total number of parts, or null if not yet known + * @return empty if valid, or an SdkClientException describing the mismatch + */ + static Optional validatePartResponse(GetObjectResponse response, + int partIndex, + long partSizeInBytes, + Long totalContentLength, + Integer totalParts) { + String contentRange = response.contentRange(); + if (contentRange == null) { + return Optional.of(missingContentRangeHeader()); + } + + Long contentLength = response.contentLength(); + if (contentLength == null || contentLength < 0) { + return Optional.of(invalidContentLength()); + } + + long expectedStartByte = partIndex * partSizeInBytes; + long[] parsedRange = MultipartDownloadUtils.parseContentRange(contentRange); + if (parsedRange == null) { + return Optional.of(invalidContentRangeHeader(contentRange)); + } + long actualStartByte = parsedRange[0]; + long actualEndByte = parsedRange[1]; + if (actualStartByte != expectedStartByte) { + return Optional.of(SdkClientException.create( + String.format("Content-Range mismatch for part %d. Expected start byte: %d, but got: bytes %d-%d", + partIndex, expectedStartByte, actualStartByte, actualEndByte))); + } + if (totalContentLength != null) { + long expectedEndByte = Math.min(expectedStartByte + partSizeInBytes - 1, totalContentLength - 1); + if (actualEndByte != expectedEndByte) { + return Optional.of(SdkClientException.create( + String.format("Content-Range mismatch for part %d. Expected: bytes %d-%d, but got: bytes %d-%d", + partIndex, expectedStartByte, expectedEndByte, actualStartByte, actualEndByte))); + } + } + + if (totalContentLength != null && totalParts != null) { + long expectedPartSize = (partIndex == totalParts - 1) + ? totalContentLength - (partIndex * partSizeInBytes) + : partSizeInBytes; + if (!contentLength.equals(expectedPartSize)) { + return Optional.of(SdkClientException.create( + String.format("Part content length validation failed for part %d. Expected: %d, but got: %d", + partIndex, expectedPartSize, contentLength))); + } + } + return Optional.empty(); + } + + /** + * Creates a range-based GET request for a specific part of a presigned URL download. + * + * @param originalRequest the original presigned URL request + * @param partIndex zero-based index of this part + * @param partSizeInBytes configured part size + * @param totalContentLength total object size, or null if not yet known (first part) + * @param eTag ETag from first response, used for If-Match on parts 1+ + * @return a new PresignedUrlDownloadRequest with the appropriate Range and If-Match headers + */ + static PresignedUrlDownloadRequest createRangedGetRequest(PresignedUrlDownloadRequest originalRequest, + int partIndex, + long partSizeInBytes, + Long totalContentLength, + String eTag) { + long startByte = partIndex * partSizeInBytes; + long endByte = totalContentLength != null + ? Math.min(startByte + partSizeInBytes - 1, totalContentLength - 1) + : startByte + partSizeInBytes - 1; + PresignedUrlDownloadRequest.Builder builder = originalRequest.toBuilder() + .range("bytes=" + startByte + "-" + endByte); + if (partIndex > 0 && eTag != null) { + builder.ifMatch(eTag); + } + return builder.build(); + } + /** * Returns true if the error is a 416 Range Not Satisfiable response from S3. * Used by subscribers to detect empty object responses on the first range request. diff --git a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/PresignedUrlMultipartDownloaderSubscriber.java b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/PresignedUrlMultipartDownloaderSubscriber.java index 2f698eea58f2..bb65bb4847db 100644 --- a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/PresignedUrlMultipartDownloaderSubscriber.java +++ b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/PresignedUrlMultipartDownloaderSubscriber.java @@ -22,7 +22,6 @@ import java.util.concurrent.atomic.AtomicInteger; import org.reactivestreams.Subscriber; import org.reactivestreams.Subscription; -import software.amazon.awssdk.annotations.Immutable; import software.amazon.awssdk.annotations.SdkInternalApi; import software.amazon.awssdk.annotations.ThreadSafe; import software.amazon.awssdk.core.async.AsyncResponseTransformer; @@ -44,13 +43,11 @@ * ordering and validation of responses.

*/ @ThreadSafe -@Immutable @SdkInternalApi public class PresignedUrlMultipartDownloaderSubscriber implements Subscriber> { private static final Logger log = Logger.loggerFor(PresignedUrlMultipartDownloaderSubscriber.class); - private static final String BYTES_RANGE_PREFIX = "bytes="; private final S3AsyncClient s3AsyncClient; private final PresignedUrlDownloadRequest presignedUrlDownloadRequest; @@ -70,7 +67,7 @@ public class PresignedUrlMultipartDownloaderSubscriber */ private final CompletableFuture resultFuture; private final Object lock = new Object(); - private final AtomicInteger completedParts; + private final AtomicInteger nextPartIndex; private final AtomicInteger requestsSent; /** @@ -91,7 +88,7 @@ public PresignedUrlMultipartDownloaderSubscriber( this.s3AsyncClient = s3AsyncClient; this.presignedUrlDownloadRequest = presignedUrlDownloadRequest; this.configuredPartSizeInBytes = configuredPartSizeInBytes; - this.completedParts = new AtomicInteger(0); + this.nextPartIndex = new AtomicInteger(0); this.requestsSent = new AtomicInteger(0); this.future = new CompletableFuture<>(); this.resultFuture = resultFuture; @@ -113,17 +110,17 @@ public void onNext(AsyncResponseTransformer= totalParts) { + currentPartIndex = nextPartIndex.get(); + if (totalParts != null && currentPartIndex >= totalParts) { log.debug(() -> String.format("Completing multipart download after a total of %d parts downloaded.", totalParts)); subscription.cancel(); return; } - completedParts.incrementAndGet(); + nextPartIndex.incrementAndGet(); } - makeRangeRequest(nextPartIndex, asyncResponseTransformer); + makeRangeRequest(currentPartIndex, asyncResponseTransformer); } private void makeRangeRequest(int partIndex, @@ -151,15 +148,15 @@ private void makeRangeRequest(int partIndex, return; } if (validatePart(response, partIndex, asyncResponseTransformer)) { - requestMoreIfNeeded(completedParts.get()); + requestMoreIfNeeded(nextPartIndex.get()); } }); } private boolean validatePart(GetObjectResponse response, int partIndex, AsyncResponseTransformer asyncResponseTransformer) { - int totalComplete = completedParts.get(); - log.debug(() -> String.format("Completed part %d", totalComplete)); + int dispatched = nextPartIndex.get(); + log.debug(() -> String.format("Dispatched %d parts so far", dispatched)); String responseETag = response.eTag(); String responseContentRange = response.contentRange(); @@ -179,7 +176,7 @@ private boolean validatePart(GetObjectResponse response, int partIndex, } this.totalContentLength = parsedContentLength.get(); - this.totalParts = calculateTotalParts(totalContentLength, configuredPartSizeInBytes); + this.totalParts = MultipartDownloadUtils.calculateTotalParts(totalContentLength, configuredPartSizeInBytes); log.debug(() -> String.format("Total content length: %d, Total parts: %d", totalContentLength, totalParts)); } @@ -194,9 +191,9 @@ private boolean validatePart(GetObjectResponse response, int partIndex, return true; } - private void requestMoreIfNeeded(int totalComplete) { + private void requestMoreIfNeeded(int dispatched) { synchronized (lock) { - if (hasMoreParts(totalComplete)) { + if (hasMoreParts(dispatched)) { subscription.request(1); } else { if (totalParts != null && requestsSent.get() != totalParts) { @@ -211,78 +208,17 @@ private void requestMoreIfNeeded(int totalComplete) { } private Optional validateResponse(GetObjectResponse response, int partIndex) { - if (response == null) { - return Optional.of(SdkClientException.create("Response cannot be null")); - } - String contentRange = response.contentRange(); - if (contentRange == null) { - return Optional.of(PresignedUrlDownloadHelper.missingContentRangeHeader()); - } - - Long contentLength = response.contentLength(); - if (contentLength == null || contentLength < 0) { - return Optional.of(PresignedUrlDownloadHelper.invalidContentLength()); - } - - long expectedStartByte = partIndex * configuredPartSizeInBytes; - long expectedEndByte; - if (totalContentLength != null) { - expectedEndByte = Math.min(expectedStartByte + configuredPartSizeInBytes - 1, totalContentLength - 1); - } else { - expectedEndByte = expectedStartByte + configuredPartSizeInBytes - 1; - } - String expectedRange = "bytes " + expectedStartByte + "-" + expectedEndByte + "/"; - if (!contentRange.startsWith(expectedRange)) { - return Optional.of(SdkClientException.create( - "Content-Range mismatch. Expected range starting with: " + expectedRange + - ", but got: " + contentRange)); - } - - long expectedPartSize; - if (totalContentLength != null && partIndex == totalParts - 1) { - expectedPartSize = totalContentLength - (partIndex * configuredPartSizeInBytes); - } else { - expectedPartSize = configuredPartSizeInBytes; - } - if (!contentLength.equals(expectedPartSize)) { - return Optional.of(SdkClientException.create( - String.format("Part content length validation failed for part %d. Expected: %d, but got: %d", - partIndex, expectedPartSize, contentLength))); - } - - long actualStartByte = MultipartDownloadUtils.parseStartByteFromContentRange(contentRange); - if (actualStartByte != expectedStartByte) { - return Optional.of(SdkClientException.create( - "Content range offset mismatch for part " + partIndex + - ". Expected start: " + expectedStartByte + ", but got: " + actualStartByte)); - } - - return Optional.empty(); + return PresignedUrlDownloadHelper.validatePartResponse( + response, partIndex, configuredPartSizeInBytes, totalContentLength, totalParts); } - private int calculateTotalParts(long contentLength, long partSize) { - return (int) Math.ceil((double) contentLength / partSize); - } - - private boolean hasMoreParts(int completedPartsCount) { - return totalParts != null && completedPartsCount < totalParts; + private boolean hasMoreParts(int dispatched) { + return totalParts != null && dispatched < totalParts; } private PresignedUrlDownloadRequest createRangedGetRequest(int partIndex) { - long startByte = partIndex * configuredPartSizeInBytes; - long endByte; - if (totalContentLength != null) { - endByte = Math.min(startByte + configuredPartSizeInBytes - 1, totalContentLength - 1); - } else { - endByte = startByte + configuredPartSizeInBytes - 1; - } - String rangeHeader = BYTES_RANGE_PREFIX + startByte + "-" + endByte; - PresignedUrlDownloadRequest.Builder builder = presignedUrlDownloadRequest.toBuilder() - .range(rangeHeader); - if (partIndex > 0 && eTag != null) { - builder.ifMatch(eTag); - } - return builder.build(); + return PresignedUrlDownloadHelper.createRangedGetRequest( + presignedUrlDownloadRequest, partIndex, configuredPartSizeInBytes, totalContentLength, eTag); } private void handleError(Throwable t) { diff --git a/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/multipart/MultipartDownloadUtilsTest.java b/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/multipart/MultipartDownloadUtilsTest.java index ef44f1d5566e..e4f04f486aa0 100644 --- a/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/multipart/MultipartDownloadUtilsTest.java +++ b/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/multipart/MultipartDownloadUtilsTest.java @@ -61,12 +61,18 @@ void contextWithParts_contextShouldBePresent() { } @Test - void parseStartByteFromContentRange_shouldParseValidAndInvalidRanges() { - assertThat(MultipartDownloadUtils.parseStartByteFromContentRange("bytes 0-1023/2048")).isEqualTo(0); - assertThat(MultipartDownloadUtils.parseStartByteFromContentRange("bytes 1024-2047/2048")).isEqualTo(1024); + void parseContentRange_shouldParseValidAndInvalidRanges() { + long[] result = MultipartDownloadUtils.parseContentRange("bytes 0-1023/2048"); + assertThat(result).isNotNull(); + assertThat(result[0]).isEqualTo(0); + assertThat(result[1]).isEqualTo(1023); - assertThat(MultipartDownloadUtils.parseStartByteFromContentRange("invalid")).isEqualTo(-1); - assertThat(MultipartDownloadUtils.parseStartByteFromContentRange(null)).isEqualTo(-1); + result = MultipartDownloadUtils.parseContentRange("bytes 1024-2047/2048"); + assertThat(result[0]).isEqualTo(1024); + assertThat(result[1]).isEqualTo(2047); + + assertThat(MultipartDownloadUtils.parseContentRange("invalid")).isNull(); + assertThat(MultipartDownloadUtils.parseContentRange(null)).isNull(); } @Test @@ -77,4 +83,13 @@ void parseContentRangeForTotalSize_shouldParseValidAndInvalidRanges() { assertThat(MultipartDownloadUtils.parseContentRangeForTotalSize("invalid")).isEmpty(); assertThat(MultipartDownloadUtils.parseContentRangeForTotalSize(null)).isEmpty(); } + + @Test + void calculateTotalParts_shouldCalculateCorrectly() { + assertThat(MultipartDownloadUtils.calculateTotalParts(32, 16)).isEqualTo(2); // exact fit + assertThat(MultipartDownloadUtils.calculateTotalParts(33, 16)).isEqualTo(3); // remainder rounds up + assertThat(MultipartDownloadUtils.calculateTotalParts(1, 16)).isEqualTo(1); // smaller than part size + assertThat(MultipartDownloadUtils.calculateTotalParts(16, 16)).isEqualTo(1); // exactly one part + assertThat(MultipartDownloadUtils.calculateTotalParts(0, 16)).isEqualTo(0); // empty object + } } \ No newline at end of file diff --git a/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/multipart/ParallelPresignedUrlMultipartDownloaderSubscriberTest.java b/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/multipart/ParallelPresignedUrlMultipartDownloaderSubscriberTest.java index e29558531418..57ace09759f0 100644 --- a/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/multipart/ParallelPresignedUrlMultipartDownloaderSubscriberTest.java +++ b/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/multipart/ParallelPresignedUrlMultipartDownloaderSubscriberTest.java @@ -336,6 +336,84 @@ void multipartObject_416OnSecondRequest_shouldFailWithError() { .isInstanceOf(CompletionException.class); } + @Test + void getObject_firstPartContentLengthMismatch_shouldFailWithValidationError() throws IOException { + stubFor(get(urlEqualTo(PRESIGNED_URL_PATH)) + .withHeader("Range", matching("bytes=0-15")) + .willReturn(aResponse() + .withStatus(206) + .withHeader("Content-Length", "10") + .withHeader("Content-Range", "bytes 0-15/32") + .withHeader("ETag", "\"test-etag\"") + .withBody(Arrays.copyOfRange(TEST_DATA, 0, 10)))); + + tempFile = createTempFileUnchecked(); + PresignedUrlDownloadRequest request = PresignedUrlDownloadRequest.builder() + .presignedUrl(presignedUrl) + .build(); + + assertThatThrownBy(() -> s3AsyncClient.presignedUrlExtension() + .getObject(request, AsyncResponseTransformer.toFile(tempFile)) + .join()) + .isInstanceOf(CompletionException.class) + .hasRootCauseInstanceOf(SdkClientException.class) + .hasMessageContaining("content length validation failed"); + } + + @Test + void getObject_firstPartContentRangeStartByteMismatch_shouldFailWithValidationError() throws IOException { + stubFor(get(urlEqualTo(PRESIGNED_URL_PATH)) + .withHeader("Range", matching("bytes=0-15")) + .willReturn(aResponse() + .withStatus(206) + .withHeader("Content-Length", "16") + .withHeader("Content-Range", "bytes 5-20/32") + .withHeader("ETag", "\"test-etag\"") + .withBody(Arrays.copyOfRange(TEST_DATA, 0, 16)))); + + tempFile = createTempFileUnchecked(); + PresignedUrlDownloadRequest request = PresignedUrlDownloadRequest.builder() + .presignedUrl(presignedUrl) + .build(); + + assertThatThrownBy(() -> s3AsyncClient.presignedUrlExtension() + .getObject(request, AsyncResponseTransformer.toFile(tempFile)) + .join()) + .isInstanceOf(CompletionException.class) + .hasRootCauseInstanceOf(SdkClientException.class) + .hasMessageContaining("Content-Range mismatch"); + } + + @Test + void getObject_objectMutatedBetweenParts_shouldFailWith412() throws IOException { + stubFor(get(urlEqualTo(PRESIGNED_URL_PATH)) + .withHeader("Range", matching("bytes=0-15")) + .willReturn(aResponse() + .withStatus(206) + .withHeader("Content-Length", "16") + .withHeader("Content-Range", "bytes 0-15/32") + .withHeader("ETag", "\"original-etag\"") + .withBody(Arrays.copyOfRange(TEST_DATA, 0, 16)))); + + stubFor(get(urlEqualTo(PRESIGNED_URL_PATH)) + .withHeader("Range", matching("bytes=16-31")) + .withHeader("If-Match", matching("\"original-etag\"")) + .willReturn(aResponse() + .withStatus(412) + .withBody("PreconditionFailed" + + "Object changed"))); + + tempFile = createTempFileUnchecked(); + PresignedUrlDownloadRequest request = PresignedUrlDownloadRequest.builder() + .presignedUrl(presignedUrl) + .build(); + + assertThatThrownBy(() -> s3AsyncClient.presignedUrlExtension() + .getObject(request, AsyncResponseTransformer.toFile(tempFile)) + .join()) + .isInstanceOf(CompletionException.class); + } + private static Path createTempFile() throws IOException { Path path = Files.createTempFile("parallel-test-" + UUID.randomUUID(), ".tmp"); Files.deleteIfExists(path); diff --git a/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/multipart/PresignedUrlDownloadHelperTest.java b/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/multipart/PresignedUrlDownloadHelperTest.java new file mode 100644 index 000000000000..ed58af0aae71 --- /dev/null +++ b/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/multipart/PresignedUrlDownloadHelperTest.java @@ -0,0 +1,182 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.services.s3.internal.multipart; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.net.MalformedURLException; +import java.net.URL; +import java.util.Optional; +import org.junit.jupiter.api.Test; +import software.amazon.awssdk.core.exception.SdkClientException; +import software.amazon.awssdk.services.s3.model.GetObjectResponse; +import software.amazon.awssdk.services.s3.presignedurl.model.PresignedUrlDownloadRequest; + +class PresignedUrlDownloadHelperTest { + + @Test + void validatePartResponse_validResponse_shouldReturnEmpty() { + GetObjectResponse response = GetObjectResponse.builder() + .contentRange("bytes 0-15/32") + .contentLength(16L) + .build(); + + Optional result = PresignedUrlDownloadHelper.validatePartResponse( + response, 0, 16L, 32L, 2); + + assertThat(result).isEmpty(); + } + + @Test + void validatePartResponse_missingContentRange_shouldReturnError() { + GetObjectResponse response = GetObjectResponse.builder() + .contentLength(16L) + .build(); + + Optional result = PresignedUrlDownloadHelper.validatePartResponse( + response, 0, 16L, 32L, 2); + + assertThat(result).isPresent(); + assertThat(result.get().getMessage()).contains("No Content-Range header"); + } + + @Test + void validatePartResponse_invalidContentLength_shouldReturnError() { + GetObjectResponse response = GetObjectResponse.builder() + .contentRange("bytes 0-15/32") + .contentLength(-1L) + .build(); + + Optional result = PresignedUrlDownloadHelper.validatePartResponse( + response, 0, 16L, 32L, 2); + + assertThat(result).isPresent(); + assertThat(result.get().getMessage()).contains("Invalid or missing Content-Length"); + } + + @Test + void validatePartResponse_contentRangeMismatch_shouldReturnError() { + GetObjectResponse response = GetObjectResponse.builder() + .contentRange("bytes 5-20/32") + .contentLength(16L) + .build(); + + Optional result = PresignedUrlDownloadHelper.validatePartResponse( + response, 0, 16L, 32L, 2); + + assertThat(result).isPresent(); + assertThat(result.get().getMessage()).contains("Content-Range mismatch for part 0"); + } + + @Test + void validatePartResponse_contentLengthMismatch_shouldReturnError() { + GetObjectResponse response = GetObjectResponse.builder() + .contentRange("bytes 0-15/32") + .contentLength(10L) + .build(); + + Optional result = PresignedUrlDownloadHelper.validatePartResponse( + response, 0, 16L, 32L, 2); + + assertThat(result).isPresent(); + assertThat(result.get().getMessage()).contains("content length validation failed"); + } + + @Test + void validatePartResponse_lastPartSmallerSize_shouldPass() { + // 30-byte object, 16-byte parts → part 1 is bytes 16-29 (14 bytes) + GetObjectResponse response = GetObjectResponse.builder() + .contentRange("bytes 16-29/30") + .contentLength(14L) + .build(); + + Optional result = PresignedUrlDownloadHelper.validatePartResponse( + response, 1, 16L, 30L, 2); + + assertThat(result).isEmpty(); + } + + @Test + void validatePartResponse_nullTotalContentLength_shouldStillValidateRange() { + GetObjectResponse response = GetObjectResponse.builder() + .contentRange("bytes 0-15/32") + .contentLength(16L) + .build(); + + // When totalContentLength is null, content-length check is skipped but range check still works + Optional result = PresignedUrlDownloadHelper.validatePartResponse( + response, 0, 16L, null, null); + + assertThat(result).isEmpty(); + } + + @Test + void createRangedGetRequest_firstPart_shouldNotIncludeIfMatch() throws MalformedURLException { + URL url = new URL("https://bucket.s3.amazonaws.com/key?X-Amz-Signature=abc"); + PresignedUrlDownloadRequest original = PresignedUrlDownloadRequest.builder() + .presignedUrl(url) + .build(); + + PresignedUrlDownloadRequest result = PresignedUrlDownloadHelper.createRangedGetRequest( + original, 0, 16L, 32L, "\"etag\""); + + assertThat(result.range()).isEqualTo("bytes=0-15"); + assertThat(result.ifMatch()).isNull(); + assertThat(result.presignedUrl()).isEqualTo(url); + } + + @Test + void createRangedGetRequest_secondPart_shouldIncludeIfMatch() throws MalformedURLException { + URL url = new URL("https://bucket.s3.amazonaws.com/key?X-Amz-Signature=abc"); + PresignedUrlDownloadRequest original = PresignedUrlDownloadRequest.builder() + .presignedUrl(url) + .build(); + + PresignedUrlDownloadRequest result = PresignedUrlDownloadHelper.createRangedGetRequest( + original, 1, 16L, 32L, "\"etag\""); + + assertThat(result.range()).isEqualTo("bytes=16-31"); + assertThat(result.ifMatch()).isEqualTo("\"etag\""); + } + + @Test + void createRangedGetRequest_lastPartClamped_shouldNotExceedTotalSize() throws MalformedURLException { + URL url = new URL("https://bucket.s3.amazonaws.com/key?X-Amz-Signature=abc"); + PresignedUrlDownloadRequest original = PresignedUrlDownloadRequest.builder() + .presignedUrl(url) + .build(); + + // 30-byte object, 16-byte parts → part 1 should be bytes=16-29 (not 16-31) + PresignedUrlDownloadRequest result = PresignedUrlDownloadHelper.createRangedGetRequest( + original, 1, 16L, 30L, "\"etag\""); + + assertThat(result.range()).isEqualTo("bytes=16-29"); + } + + @Test + void createRangedGetRequest_nullTotalContentLength_shouldUseFullPartSize() throws MalformedURLException { + URL url = new URL("https://bucket.s3.amazonaws.com/key?X-Amz-Signature=abc"); + PresignedUrlDownloadRequest original = PresignedUrlDownloadRequest.builder() + .presignedUrl(url) + .build(); + + // First part when total size unknown — uses full part size without clamping + PresignedUrlDownloadRequest result = PresignedUrlDownloadHelper.createRangedGetRequest( + original, 0, 16L, null, null); + + assertThat(result.range()).isEqualTo("bytes=0-15"); + } +} From bfaea56ad00135eac51b26d616ceeb9e7aae0508 Mon Sep 17 00:00:00 2001 From: jencymaryjoseph <35571282+jencymaryjoseph@users.noreply.github.com> Date: Mon, 15 Jun 2026 16:52:33 -0700 Subject: [PATCH 23/30] Fix issues in parts calculation (#7026) --- .../multipart/MultipartDownloadUtils.java | 5 +- ...ignedUrlMultipartDownloaderSubscriber.java | 48 +++++++++---------- .../multipart/PresignedUrlDownloadHelper.java | 6 +-- ...ignedUrlMultipartDownloaderSubscriber.java | 28 +++++------ .../multipart/MultipartDownloadUtilsTest.java | 9 +++- .../PresignedUrlDownloadHelperTest.java | 12 ++--- 6 files changed, 58 insertions(+), 50 deletions(-) diff --git a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/MultipartDownloadUtils.java b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/MultipartDownloadUtils.java index cf1568b7a432..c7e9885e34d3 100644 --- a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/MultipartDownloadUtils.java +++ b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/MultipartDownloadUtils.java @@ -121,8 +121,9 @@ public static Optional parseContentRangeForTotalSize(String contentRange) * @param partSize size of each part in bytes * @return the number of parts */ - public static int calculateTotalParts(long contentLength, long partSize) { - return (int) Math.ceil((double) contentLength / partSize); + public static long calculateTotalParts(long contentLength, long partSize) { + return (contentLength / partSize) + (contentLength % partSize == 0 ? 0 : 1); + } } diff --git a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/ParallelPresignedUrlMultipartDownloaderSubscriber.java b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/ParallelPresignedUrlMultipartDownloaderSubscriber.java index 5c62778d85ee..88031de5c733 100644 --- a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/ParallelPresignedUrlMultipartDownloaderSubscriber.java +++ b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/ParallelPresignedUrlMultipartDownloaderSubscriber.java @@ -23,7 +23,7 @@ import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.Semaphore; import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; import org.reactivestreams.Subscriber; import org.reactivestreams.Subscription; import software.amazon.awssdk.annotations.SdkInternalApi; @@ -62,23 +62,23 @@ public class ParallelPresignedUrlMultipartDownloaderSubscriber private final CompletableFuture resultFuture; private final int maxInFlightParts; - private final AtomicInteger partNumber = new AtomicInteger(0); - private final AtomicInteger completedParts = new AtomicInteger(0); + private final AtomicLong partNumber = new AtomicLong(0); + private final AtomicLong completedParts = new AtomicLong(0); private final Semaphore inFlightPermits; /** * CAS gate ensuring only the first part failure triggers error handling and cancellation. */ private final AtomicBoolean downloadFailed = new AtomicBoolean(false); private final AtomicBoolean processingPending = new AtomicBoolean(false); - private final Map> inFlightRequests = new ConcurrentHashMap<>(); - private final Queue>> pendingTransformers = + private final Map> inFlightRequests = new ConcurrentHashMap<>(); + private final Queue>> pendingTransformers = new ConcurrentLinkedQueue<>(); private final Object subscriptionLock = new Object(); private Subscription subscription; private volatile Long totalContentLength; - private volatile Integer totalParts; + private volatile Long totalParts; private volatile String eTag; private volatile GetObjectResponse firstResponse; @@ -112,7 +112,7 @@ public void onNext(AsyncResponseTransformer transformer) { - PresignedUrlDownloadRequest partRequest = createRangedGetRequest(0); + PresignedUrlDownloadRequest partRequest = createRangedGetRequest(0L); log.debug(() -> "Sending first range request with range=" + partRequest.range()); if (!inFlightPermits.tryAcquire()) { @@ -139,11 +139,11 @@ private void sendFirstRequest(AsyncResponseTransformer response = s3AsyncClient.presignedUrlExtension().getObject(partRequest, transformer); - inFlightRequests.put(0, response); + inFlightRequests.put(0L, response); CompletableFutureUtils.forwardExceptionTo(resultFuture, response); response.whenComplete((res, error) -> { - inFlightRequests.remove(0); + inFlightRequests.remove(0L); inFlightPermits.release(); if (error != null) { @@ -154,7 +154,7 @@ private void sendFirstRequest(AsyncResponseTransformer parsedTotal = MultipartDownloadUtils.parseContentRangeForTotalSize(contentRange); if (!parsedTotal.isPresent()) { - handlePartError(PresignedUrlDownloadHelper.invalidContentRangeHeader(contentRange), 0); + handlePartError(PresignedUrlDownloadHelper.invalidContentRangeHeader(contentRange), 0L); return; } @@ -184,9 +184,9 @@ private void sendFirstRequest(AsyncResponseTransformer String.format("Total content length: %d, Total parts: %d", totalContentLength, totalParts)); - Optional validationError = validatePartResponse(res, 0); + Optional validationError = validatePartResponse(res, 0L); if (validationError.isPresent()) { - handlePartError(validationError.get(), 0); + handlePartError(validationError.get(), 0L); return; } @@ -200,8 +200,8 @@ private void sendFirstRequest(AsyncResponseTransformer transformer, - int currentPart) { + long currentPart) { if (currentPart >= totalParts) { return; } @@ -224,7 +224,7 @@ private void processRequest(AsyncResponseTransformer transformer, - int partIndex) { + long partIndex) { if (downloadFailed.get()) { inFlightPermits.release(); return; @@ -259,7 +259,7 @@ private void sendPartRequest(AsyncResponseTransformer "Completed part: " + partIndex); - int totalComplete = completedParts.incrementAndGet(); + long totalComplete = completedParts.incrementAndGet(); if (totalComplete == totalParts) { resultFuture.complete(firstResponse); @@ -285,7 +285,7 @@ private void processPendingTransformers() { try { // Drain pending queue while permits are available while (!pendingTransformers.isEmpty() && inFlightPermits.tryAcquire()) { - Pair> pendingPart = + Pair> pendingPart = pendingTransformers.poll(); if (pendingPart != null && pendingPart.left() < totalParts) { sendPartRequest(pendingPart.right(), pendingPart.left()); @@ -299,12 +299,12 @@ private void processPendingTransformers() { } while (!pendingTransformers.isEmpty() && inFlightPermits.availablePermits() > 0); } - private Optional validatePartResponse(GetObjectResponse response, int partIndex) { + private Optional validatePartResponse(GetObjectResponse response, long partIndex) { return PresignedUrlDownloadHelper.validatePartResponse( response, partIndex, configuredPartSizeInBytes, totalContentLength, totalParts); } - private void handlePartError(Throwable error, int partIndex) { + private void handlePartError(Throwable error, long partIndex) { if (downloadFailed.compareAndSet(false, true)) { log.debug(() -> "Error on part " + partIndex, error); resultFuture.completeExceptionally(error); @@ -317,7 +317,7 @@ private void handlePartError(Throwable error, int partIndex) { } } - private PresignedUrlDownloadRequest createRangedGetRequest(int partIndex) { + private PresignedUrlDownloadRequest createRangedGetRequest(long partIndex) { return PresignedUrlDownloadHelper.createRangedGetRequest( presignedUrlDownloadRequest, partIndex, configuredPartSizeInBytes, totalContentLength, eTag); } diff --git a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/PresignedUrlDownloadHelper.java b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/PresignedUrlDownloadHelper.java index 9cc521fd98f2..77beddae9949 100644 --- a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/PresignedUrlDownloadHelper.java +++ b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/PresignedUrlDownloadHelper.java @@ -155,10 +155,10 @@ static SdkClientException invalidContentLength() { * @return empty if valid, or an SdkClientException describing the mismatch */ static Optional validatePartResponse(GetObjectResponse response, - int partIndex, + long partIndex, long partSizeInBytes, Long totalContentLength, - Integer totalParts) { + Long totalParts) { String contentRange = response.contentRange(); if (contentRange == null) { return Optional.of(missingContentRangeHeader()); @@ -214,7 +214,7 @@ static Optional validatePartResponse(GetObjectResponse respo * @return a new PresignedUrlDownloadRequest with the appropriate Range and If-Match headers */ static PresignedUrlDownloadRequest createRangedGetRequest(PresignedUrlDownloadRequest originalRequest, - int partIndex, + long partIndex, long partSizeInBytes, Long totalContentLength, String eTag) { diff --git a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/PresignedUrlMultipartDownloaderSubscriber.java b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/PresignedUrlMultipartDownloaderSubscriber.java index bb65bb4847db..86c4c76308ca 100644 --- a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/PresignedUrlMultipartDownloaderSubscriber.java +++ b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/PresignedUrlMultipartDownloaderSubscriber.java @@ -19,7 +19,7 @@ import java.util.Queue; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentLinkedQueue; -import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; import org.reactivestreams.Subscriber; import org.reactivestreams.Subscription; import software.amazon.awssdk.annotations.SdkInternalApi; @@ -67,8 +67,8 @@ public class PresignedUrlMultipartDownloaderSubscriber */ private final CompletableFuture resultFuture; private final Object lock = new Object(); - private final AtomicInteger nextPartIndex; - private final AtomicInteger requestsSent; + private final AtomicLong nextPartIndex; + private final AtomicLong requestsSent; /** * Store the GetObject futures so we can cancel them if onError() is invoked. @@ -76,7 +76,7 @@ public class PresignedUrlMultipartDownloaderSubscriber private final Queue> getObjectFutures = new ConcurrentLinkedQueue<>(); private volatile Long totalContentLength; - private volatile Integer totalParts; + private volatile Long totalParts; private volatile String eTag; private Subscription subscription; @@ -88,8 +88,8 @@ public PresignedUrlMultipartDownloaderSubscriber( this.s3AsyncClient = s3AsyncClient; this.presignedUrlDownloadRequest = presignedUrlDownloadRequest; this.configuredPartSizeInBytes = configuredPartSizeInBytes; - this.nextPartIndex = new AtomicInteger(0); - this.requestsSent = new AtomicInteger(0); + this.nextPartIndex = new AtomicLong(0); + this.requestsSent = new AtomicLong(0); this.future = new CompletableFuture<>(); this.resultFuture = resultFuture; } @@ -110,7 +110,7 @@ public void onNext(AsyncResponseTransformer= totalParts) { @@ -123,7 +123,7 @@ public void onNext(AsyncResponseTransformer asyncResponseTransformer) { PresignedUrlDownloadRequest partRequest = createRangedGetRequest(partIndex); @@ -153,9 +153,9 @@ private void makeRangeRequest(int partIndex, }); } - private boolean validatePart(GetObjectResponse response, int partIndex, + private boolean validatePart(GetObjectResponse response, long partIndex, AsyncResponseTransformer asyncResponseTransformer) { - int dispatched = nextPartIndex.get(); + long dispatched = nextPartIndex.get(); log.debug(() -> String.format("Dispatched %d parts so far", dispatched)); String responseETag = response.eTag(); @@ -191,7 +191,7 @@ private boolean validatePart(GetObjectResponse response, int partIndex, return true; } - private void requestMoreIfNeeded(int dispatched) { + private void requestMoreIfNeeded(long dispatched) { synchronized (lock) { if (hasMoreParts(dispatched)) { subscription.request(1); @@ -207,16 +207,16 @@ private void requestMoreIfNeeded(int dispatched) { } } - private Optional validateResponse(GetObjectResponse response, int partIndex) { + private Optional validateResponse(GetObjectResponse response, long partIndex) { return PresignedUrlDownloadHelper.validatePartResponse( response, partIndex, configuredPartSizeInBytes, totalContentLength, totalParts); } - private boolean hasMoreParts(int dispatched) { + private boolean hasMoreParts(long dispatched) { return totalParts != null && dispatched < totalParts; } - private PresignedUrlDownloadRequest createRangedGetRequest(int partIndex) { + private PresignedUrlDownloadRequest createRangedGetRequest(long partIndex) { return PresignedUrlDownloadHelper.createRangedGetRequest( presignedUrlDownloadRequest, partIndex, configuredPartSizeInBytes, totalContentLength, eTag); } diff --git a/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/multipart/MultipartDownloadUtilsTest.java b/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/multipart/MultipartDownloadUtilsTest.java index e4f04f486aa0..acc535b686eb 100644 --- a/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/multipart/MultipartDownloadUtilsTest.java +++ b/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/multipart/MultipartDownloadUtilsTest.java @@ -91,5 +91,12 @@ void calculateTotalParts_shouldCalculateCorrectly() { assertThat(MultipartDownloadUtils.calculateTotalParts(1, 16)).isEqualTo(1); // smaller than part size assertThat(MultipartDownloadUtils.calculateTotalParts(16, 16)).isEqualTo(1); // exactly one part assertThat(MultipartDownloadUtils.calculateTotalParts(0, 16)).isEqualTo(0); // empty object + // 5 GiB / 1 byte = 5_368_709_120 parts (exceeds Integer.MAX_VALUE) + long fiveGiB = 5L * 1024L * 1024L * 1024L; + assertThat(MultipartDownloadUtils.calculateTotalParts(fiveGiB, 1L)).isEqualTo(fiveGiB); + assertThat(MultipartDownloadUtils.calculateTotalParts(Long.MAX_VALUE - 1, 1L)) + .isEqualTo(Long.MAX_VALUE - 1); + assertThat(MultipartDownloadUtils.calculateTotalParts(Long.MAX_VALUE, 2)) + .isEqualTo((Long.MAX_VALUE / 2) + 1); } -} \ No newline at end of file +} diff --git a/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/multipart/PresignedUrlDownloadHelperTest.java b/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/multipart/PresignedUrlDownloadHelperTest.java index ed58af0aae71..7902916abf97 100644 --- a/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/multipart/PresignedUrlDownloadHelperTest.java +++ b/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/multipart/PresignedUrlDownloadHelperTest.java @@ -35,7 +35,7 @@ void validatePartResponse_validResponse_shouldReturnEmpty() { .build(); Optional result = PresignedUrlDownloadHelper.validatePartResponse( - response, 0, 16L, 32L, 2); + response, 0, 16L, 32L, 2L); assertThat(result).isEmpty(); } @@ -47,7 +47,7 @@ void validatePartResponse_missingContentRange_shouldReturnError() { .build(); Optional result = PresignedUrlDownloadHelper.validatePartResponse( - response, 0, 16L, 32L, 2); + response, 0, 16L, 32L, 2L); assertThat(result).isPresent(); assertThat(result.get().getMessage()).contains("No Content-Range header"); @@ -61,7 +61,7 @@ void validatePartResponse_invalidContentLength_shouldReturnError() { .build(); Optional result = PresignedUrlDownloadHelper.validatePartResponse( - response, 0, 16L, 32L, 2); + response, 0, 16L, 32L, 2L); assertThat(result).isPresent(); assertThat(result.get().getMessage()).contains("Invalid or missing Content-Length"); @@ -75,7 +75,7 @@ void validatePartResponse_contentRangeMismatch_shouldReturnError() { .build(); Optional result = PresignedUrlDownloadHelper.validatePartResponse( - response, 0, 16L, 32L, 2); + response, 0, 16L, 32L, 2L); assertThat(result).isPresent(); assertThat(result.get().getMessage()).contains("Content-Range mismatch for part 0"); @@ -89,7 +89,7 @@ void validatePartResponse_contentLengthMismatch_shouldReturnError() { .build(); Optional result = PresignedUrlDownloadHelper.validatePartResponse( - response, 0, 16L, 32L, 2); + response, 0, 16L, 32L, 2L); assertThat(result).isPresent(); assertThat(result.get().getMessage()).contains("content length validation failed"); @@ -104,7 +104,7 @@ void validatePartResponse_lastPartSmallerSize_shouldPass() { .build(); Optional result = PresignedUrlDownloadHelper.validatePartResponse( - response, 1, 16L, 30L, 2); + response, 1, 16L, 30L, 2L); assertThat(result).isEmpty(); } From 014ea4b287cc1a284b23dc3d61bfb31e47e68b38 Mon Sep 17 00:00:00 2001 From: jencymaryjoseph <35571282+jencymaryjoseph@users.noreply.github.com> Date: Wed, 17 Jun 2026 15:39:26 -0700 Subject: [PATCH 24/30] Enable checksum validation for presigned URL downloads (#7035) * Enable checksum validation for presigned URL downloads * Add public method to return all checksum algorithms * assert checksum validation status --- .../checksums/DefaultChecksumAlgorithm.java | 9 + .../AsyncPresignedUrlExtensionTestSuite.java | 56 +++++ .../handlers/GetObjectInterceptor.java | 4 +- .../DefaultAsyncPresignedUrlExtension.java | 30 ++- ...PresignedUrlDownloadRequestMarshaller.java | 39 +++- .../AsyncPresignedUrlExtension.java | 8 + .../services/s3/presigner/S3Presigner.java | 6 + ...DefaultAsyncPresignedUrlExtensionTest.java | 65 ++++++ ...gnedUrlChecksumValidationWiremockTest.java | 216 ++++++++++++++++++ ...ignedUrlDownloadRequestMarshallerTest.java | 63 +++++ 10 files changed, 489 insertions(+), 7 deletions(-) create mode 100644 services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/presignedurl/PresignedUrlChecksumValidationWiremockTest.java diff --git a/core/checksums/src/main/java/software/amazon/awssdk/checksums/DefaultChecksumAlgorithm.java b/core/checksums/src/main/java/software/amazon/awssdk/checksums/DefaultChecksumAlgorithm.java index 65a2940b8aa2..dae3f27db564 100644 --- a/core/checksums/src/main/java/software/amazon/awssdk/checksums/DefaultChecksumAlgorithm.java +++ b/core/checksums/src/main/java/software/amazon/awssdk/checksums/DefaultChecksumAlgorithm.java @@ -15,6 +15,8 @@ package software.amazon.awssdk.checksums; +import java.util.Collection; +import java.util.Collections; import java.util.Locale; import java.util.concurrent.ConcurrentHashMap; import software.amazon.awssdk.annotations.SdkProtectedApi; @@ -52,6 +54,13 @@ public static ChecksumAlgorithm fromValue(String algorithm) { return ChecksumAlgorithmsCache.VALUES.get(algorithm.toUpperCase(Locale.US)); } + /** + * Returns all registered checksum algorithms. + */ + public static Collection values() { + return Collections.unmodifiableCollection(ChecksumAlgorithmsCache.VALUES.values()); + } + private static final class ChecksumAlgorithmsCache { private static final ConcurrentHashMap VALUES = new ConcurrentHashMap<>(); diff --git a/services/s3/src/it/java/software/amazon/awssdk/services/s3/presignedurl/AsyncPresignedUrlExtensionTestSuite.java b/services/s3/src/it/java/software/amazon/awssdk/services/s3/presignedurl/AsyncPresignedUrlExtensionTestSuite.java index d32c7038317a..1ff737b52850 100644 --- a/services/s3/src/it/java/software/amazon/awssdk/services/s3/presignedurl/AsyncPresignedUrlExtensionTestSuite.java +++ b/services/s3/src/it/java/software/amazon/awssdk/services/s3/presignedurl/AsyncPresignedUrlExtensionTestSuite.java @@ -45,6 +45,7 @@ import software.amazon.awssdk.metrics.MetricPublisher; import software.amazon.awssdk.services.s3.S3AsyncClient; import software.amazon.awssdk.services.s3.S3IntegrationTestBase; +import software.amazon.awssdk.services.s3.model.ChecksumMode; import software.amazon.awssdk.services.s3.model.DeleteObjectRequest; import software.amazon.awssdk.services.s3.model.GetObjectResponse; import software.amazon.awssdk.services.s3.model.PutObjectRequest; @@ -280,6 +281,61 @@ void getObject_emptyObject_withRange_shouldThrow416() throws Exception { .hasCauseInstanceOf(S3Exception.class); } + @Test + void getObject_withChecksumModeEnabled_returnsChecksumHeaders() throws Exception { + PresignedGetObjectRequest presigned = presigner.presignGetObject(r -> r + .getObjectRequest(req -> req.bucket(testBucket).key(testGetObjectKey) + .checksumMode(ChecksumMode.ENABLED)) + .signatureDuration(Duration.ofMinutes(10))); + + PresignedUrlDownloadRequest request = PresignedUrlDownloadRequest.builder() + .presignedUrl(presigned.url()) + .build(); + + ResponseBytes response = + presignedUrlExtension.getObject(request, AsyncResponseTransformer.toBytes()) + .get(30, TimeUnit.SECONDS); + + assertThat(response.asUtf8String()).isEqualTo(testObjectContent); + assertThat(response.response().checksumTypeAsString()).isNotNull(); + } + + @Test + void getObject_withoutChecksumMode_doesNotReturnChecksumHeaders() throws Exception { + PresignedUrlDownloadRequest request = createRequestForKey(testGetObjectKey); + + ResponseBytes response = + presignedUrlExtension.getObject(request, AsyncResponseTransformer.toBytes()) + .get(30, TimeUnit.SECONDS); + + assertThat(response.asUtf8String()).isEqualTo(testObjectContent); + assertThat(response.response().checksumTypeAsString()).isNull(); + } + + @Test + void getObject_withChecksumModeEnabled_requestContainsChecksumHeader() throws Exception { + PresignedGetObjectRequest presigned = presigner.presignGetObject(r -> r + .getObjectRequest(req -> req.bucket(testBucket).key(testGetObjectKey) + .checksumMode(ChecksumMode.ENABLED)) + .signatureDuration(Duration.ofMinutes(10))); + + // Verify the presigned URL has checksum-mode in SignedHeaders + assertThat(presigned.signedHeaders()).containsKey("x-amz-checksum-mode"); + assertThat(presigned.isBrowserExecutable()).isFalse(); + + PresignedUrlDownloadRequest request = PresignedUrlDownloadRequest.builder() + .presignedUrl(presigned.url()) + .build(); + + // Download should succeed (proves marshaller sent the header) + ResponseBytes response = + presignedUrlExtension.getObject(request, AsyncResponseTransformer.toBytes()) + .get(30, TimeUnit.SECONDS); + + assertThat(response).isNotNull(); + assertThat(response.asUtf8String()).isEqualTo(testObjectContent); + } + static Stream basicFunctionalityTestData() { return Stream.of( Arguments.of("getObject_withValidUrl_returnsContent", diff --git a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/handlers/GetObjectInterceptor.java b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/handlers/GetObjectInterceptor.java index 7e13f6b97473..5929765d3b29 100644 --- a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/handlers/GetObjectInterceptor.java +++ b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/handlers/GetObjectInterceptor.java @@ -28,6 +28,7 @@ import software.amazon.awssdk.core.interceptor.SdkExecutionAttribute; import software.amazon.awssdk.core.internal.util.HttpChecksumUtils; import software.amazon.awssdk.http.SdkHttpResponse; +import software.amazon.awssdk.services.s3.internal.presignedurl.model.PresignedUrlDownloadRequestWrapper; import software.amazon.awssdk.services.s3.model.GetObjectRequest; import software.amazon.awssdk.services.s3.model.GetObjectResponse; import software.amazon.awssdk.utils.Pair; @@ -43,7 +44,8 @@ public class GetObjectInterceptor implements ExecutionInterceptor { @Override public void afterTransmission(Context.AfterTransmission context, ExecutionAttributes executionAttributes) { - if (!(context.request() instanceof GetObjectRequest)) { + if (!(context.request() instanceof GetObjectRequest) + && !(context.request() instanceof PresignedUrlDownloadRequestWrapper)) { return; } ChecksumSpecs resolvedChecksumSpecs = executionAttributes.getAttribute(SdkExecutionAttribute.RESOLVED_CHECKSUM_SPECS); diff --git a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/presignedurl/DefaultAsyncPresignedUrlExtension.java b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/presignedurl/DefaultAsyncPresignedUrlExtension.java index d640482c5c36..d2a7088aa865 100644 --- a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/presignedurl/DefaultAsyncPresignedUrlExtension.java +++ b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/presignedurl/DefaultAsyncPresignedUrlExtension.java @@ -27,6 +27,8 @@ import software.amazon.awssdk.annotations.SdkInternalApi; import software.amazon.awssdk.awscore.exception.AwsServiceException; import software.amazon.awssdk.awscore.internal.AwsProtocolMetadata; +import software.amazon.awssdk.checksums.DefaultChecksumAlgorithm; +import software.amazon.awssdk.checksums.spi.ChecksumAlgorithm; import software.amazon.awssdk.core.async.AsyncResponseTransformer; import software.amazon.awssdk.core.async.AsyncResponseTransformerUtils; import software.amazon.awssdk.core.client.config.SdkAdvancedClientOption; @@ -37,6 +39,7 @@ import software.amazon.awssdk.core.exception.SdkClientException; import software.amazon.awssdk.core.http.HttpResponseHandler; import software.amazon.awssdk.core.interceptor.SdkInternalExecutionAttribute; +import software.amazon.awssdk.core.interceptor.trait.HttpChecksum; import software.amazon.awssdk.core.metrics.CoreMetric; import software.amazon.awssdk.core.signer.NoOpSigner; import software.amazon.awssdk.metrics.MetricCollector; @@ -61,12 +64,24 @@ public final class DefaultAsyncPresignedUrlExtension implements AsyncPresignedUrlExtension { private static final Logger log = LoggerFactory.getLogger(DefaultAsyncPresignedUrlExtension.class); + /** + * Checksum configuration matching the codegen-produced GetObject operation. + * Enables the HttpChecksumValidationInterceptor to validate response checksums. + */ + private static final HttpChecksum RESPONSE_CHECKSUM_CONFIG = + HttpChecksum.builder() + .requestValidationMode("ENABLED") + .responseAlgorithmsV2( + DefaultChecksumAlgorithm.values() + .toArray(new ChecksumAlgorithm[0])) + .build(); + private final AsyncClientHandler clientHandler; private final AwsS3ProtocolFactory protocolFactory; private final SdkClientConfiguration clientConfiguration; private final List metricPublishers; private final AwsProtocolMetadata protocolMetadata; - + public DefaultAsyncPresignedUrlExtension(AsyncClientHandler clientHandler, AwsS3ProtocolFactory protocolFactory, SdkClientConfiguration clientConfiguration, @@ -122,6 +137,8 @@ public CompletableFuture getObject( .withMetricCollector(apiCallMetricCollector) // TODO: Deprecate IS_DISCOVERED_ENDPOINT, use new SKIP_ENDPOINT_RESOLUTION for better semantics .putExecutionAttribute(SdkInternalExecutionAttribute.IS_DISCOVERED_ENDPOINT, true) + .putExecutionAttribute(SdkInternalExecutionAttribute.HTTP_CHECKSUM, + checksumConfigForUrl(presignedUrlDownloadRequest.presignedUrl())) .withMarshaller(new PresignedUrlDownloadRequestMarshaller(protocolFactory)), asyncResponseTransformer); @@ -151,4 +168,15 @@ private SdkClientConfiguration updateSdkClientConfiguration(SdkClientConfigurati return configBuilder.build(); } + /** + * Returns the checksum validation config if the presigned URL was signed with checksum mode, + * or null otherwise. + */ + private static HttpChecksum checksumConfigForUrl(java.net.URL presignedUrl) { + if (PresignedUrlDownloadRequestMarshaller.hasChecksumModeInSignedHeaders(presignedUrl.getQuery())) { + return RESPONSE_CHECKSUM_CONFIG; + } + return null; + } + } diff --git a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/presignedurl/PresignedUrlDownloadRequestMarshaller.java b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/presignedurl/PresignedUrlDownloadRequestMarshaller.java index f0aecfce87da..b37aec335184 100644 --- a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/presignedurl/PresignedUrlDownloadRequestMarshaller.java +++ b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/presignedurl/PresignedUrlDownloadRequestMarshaller.java @@ -63,14 +63,43 @@ public SdkHttpFullRequest marshall(PresignedUrlDownloadRequestWrapper presignedU .createProtocolMarshaller(SDK_OPERATION_BINDING); URI presignedUri = presignedUrlDownloadRequestWrapper.url().toURI(); - return protocolMarshaller.marshall(presignedUrlDownloadRequestWrapper) - .toBuilder() - .uri(presignedUri) - .build(); + SdkHttpFullRequest.Builder requestBuilder = protocolMarshaller + .marshall(presignedUrlDownloadRequestWrapper) + .toBuilder() + .uri(presignedUri); + + addChecksumModeHeaderIfSignedInUrl(requestBuilder, presignedUri); + + return requestBuilder.build(); } catch (Exception e) { throw SdkClientException.builder() .message("Unable to marshall pre-signed URL Request: " + e.getMessage()) .cause(e).build(); } } -} \ No newline at end of file + + /** + * If the presigned URL's X-Amz-SignedHeaders contains "x-amz-checksum-mode", automatically add + * the header with value "ENABLED" so S3 returns checksum headers in the response. + */ + private void addChecksumModeHeaderIfSignedInUrl(SdkHttpFullRequest.Builder requestBuilder, URI uri) { + if (hasChecksumModeInSignedHeaders(uri.getQuery())) { + requestBuilder.putHeader("x-amz-checksum-mode", "ENABLED"); + } + } + + /** + * Returns true if the decoded query string's X-Amz-SignedHeaders parameter contains "x-amz-checksum-mode". + */ + static boolean hasChecksumModeInSignedHeaders(String query) { + if (query == null) { + return false; + } + for (String param : query.split("&")) { + if (param.startsWith("X-Amz-SignedHeaders=")) { + return param.substring("X-Amz-SignedHeaders=".length()).contains("x-amz-checksum-mode"); + } + } + return false; + } +} diff --git a/services/s3/src/main/java/software/amazon/awssdk/services/s3/presignedurl/AsyncPresignedUrlExtension.java b/services/s3/src/main/java/software/amazon/awssdk/services/s3/presignedurl/AsyncPresignedUrlExtension.java index 545eb838c35e..11103a101580 100644 --- a/services/s3/src/main/java/software/amazon/awssdk/services/s3/presignedurl/AsyncPresignedUrlExtension.java +++ b/services/s3/src/main/java/software/amazon/awssdk/services/s3/presignedurl/AsyncPresignedUrlExtension.java @@ -29,6 +29,14 @@ /** * Interface for executing S3 operations asynchronously using presigned URLs. This can be accessed using * {@link S3AsyncClient#presignedUrlExtension()}. + * + *

Checksum Validation: If the presigned URL was generated with + * {@link software.amazon.awssdk.services.s3.presigner.S3Presigner#presignGetObject} using + * {@code checksumMode(ChecksumMode.ENABLED)}, the SDK automatically sends the required header + * and S3 returns checksums for full object downloads (HTTP 200). For ranged downloads (HTTP 206), + * checksums are only returned for multipart-uploaded objects when the range aligns with original + * upload part boundaries. The downloader cannot enable checksums if the URL was not presigned + * with checksum mode. */ @SdkPublicApi public interface AsyncPresignedUrlExtension { diff --git a/services/s3/src/main/java/software/amazon/awssdk/services/s3/presigner/S3Presigner.java b/services/s3/src/main/java/software/amazon/awssdk/services/s3/presigner/S3Presigner.java index e23516178e92..b5fd634255fa 100644 --- a/services/s3/src/main/java/software/amazon/awssdk/services/s3/presigner/S3Presigner.java +++ b/services/s3/src/main/java/software/amazon/awssdk/services/s3/presigner/S3Presigner.java @@ -294,6 +294,12 @@ static Builder builder() { * signing or authentication. *

* + * Checksum support: Setting {@code checksumMode(ChecksumMode.ENABLED)} on the + * {@code GetObjectRequest} enables the downloader to receive checksums from S3 for data integrity + * validation. The resulting URL will not be browser-executable (requires the + * {@code x-amz-checksum-mode} header at download time, which the SDK sends automatically). + *

+ * * Example Usage *

* diff --git a/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/presignedurl/DefaultAsyncPresignedUrlExtensionTest.java b/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/presignedurl/DefaultAsyncPresignedUrlExtensionTest.java index 91681fbb8e15..9fed4335aa5a 100644 --- a/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/presignedurl/DefaultAsyncPresignedUrlExtensionTest.java +++ b/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/presignedurl/DefaultAsyncPresignedUrlExtensionTest.java @@ -341,6 +341,71 @@ private void assertSuccessfulGetObject(PresignedUrlDownloadRequest request) { } } + @ParameterizedTest(name = "{0}") + @MethodSource("checksumValidationTestCases") + void getObject_withChecksumHeaderInResponse_shouldDownloadSuccessfully( + String testName, + HttpExecuteResponse response, + String testUrl, + boolean expectSuccess) throws Exception { + + mockHttpClient.stubNextResponse(response); + + URL presignedUrl = new URL(testUrl); + PresignedUrlDownloadRequest request = PresignedUrlDownloadRequest.builder() + .presignedUrl(presignedUrl) + .build(); + + CompletableFuture> future = + presignedUrlExtension.getObject(request, AsyncResponseTransformer.toBytes()); + + ResponseBytes result = future.get(); + assertThat(result).isNotNull(); + assertThat(result.asUtf8String()).isEqualTo(TEST_CONTENT); + } + + private static Stream checksumValidationTestCases() { + // CRC32 of "test-content" = 0x6B59FCDE → base64 = a1n83g== + String correctCrc32 = "a1n83g=="; + String urlWithChecksumSigned = "https://test-bucket.s3.us-east-1.amazonaws.com/test-key?" + + "X-Amz-Algorithm=AWS4-HMAC-SHA256&" + + "X-Amz-Date=20250707T000000Z&" + + "X-Amz-SignedHeaders=host%3Bx-amz-checksum-mode&" + + "X-Amz-Signature=test-signature&" + + "X-Amz-Credential=AKIAIOSFODNN7EXAMPLE%2F20250707%2Fus-east-1%2Fs3%2Faws4_request&" + + "X-Amz-Expires=600"; + + return Stream.of( + Arguments.of( + "With checksum in response - should succeed", + createResponseWithChecksum("x-amz-checksum-crc32", correctCrc32), + urlWithChecksumSigned, + true + ), + Arguments.of( + "No checksum header in response - should succeed without validation", + createSuccessResponse(), + TEST_URL, + true + ) + ); + } + + private static HttpExecuteResponse createResponseWithChecksum(String checksumHeader, String checksumValue) { + SdkHttpFullResponse httpResponse = SdkHttpFullResponse.builder() + .statusCode(200) + .putHeader("Content-Length", "12") + .putHeader("ETag", "\"test-etag\"") + .putHeader("Content-Type", "text/plain") + .putHeader(checksumHeader, checksumValue) + .build(); + return HttpExecuteResponse.builder() + .response(httpResponse) + .responseBody(AbortableInputStream.create( + new ByteArrayInputStream(TEST_CONTENT.getBytes(StandardCharsets.UTF_8)))) + .build(); + } + private static HttpExecuteResponse createSuccessResponse() { SdkHttpFullResponse httpResponse = SdkHttpFullResponse.builder() .statusCode(200) diff --git a/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/presignedurl/PresignedUrlChecksumValidationWiremockTest.java b/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/presignedurl/PresignedUrlChecksumValidationWiremockTest.java new file mode 100644 index 000000000000..fc2a7a26cf97 --- /dev/null +++ b/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/presignedurl/PresignedUrlChecksumValidationWiremockTest.java @@ -0,0 +1,216 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.services.s3.internal.presignedurl; + +import static com.github.tomakehurst.wiremock.client.WireMock.aResponse; +import static com.github.tomakehurst.wiremock.client.WireMock.get; +import static com.github.tomakehurst.wiremock.client.WireMock.stubFor; +import static com.github.tomakehurst.wiremock.client.WireMock.urlPathEqualTo; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import com.github.tomakehurst.wiremock.junit5.WireMockRuntimeInfo; +import com.github.tomakehurst.wiremock.junit5.WireMockTest; +import java.net.URL; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.atomic.AtomicReference; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import software.amazon.awssdk.auth.credentials.AnonymousCredentialsProvider; +import software.amazon.awssdk.core.ResponseBytes; +import software.amazon.awssdk.core.async.AsyncResponseTransformer; +import software.amazon.awssdk.core.checksums.ChecksumValidation; +import software.amazon.awssdk.core.exception.SdkClientException; +import software.amazon.awssdk.core.interceptor.Context; +import software.amazon.awssdk.core.interceptor.ExecutionAttributes; +import software.amazon.awssdk.core.interceptor.ExecutionInterceptor; +import software.amazon.awssdk.core.interceptor.SdkExecutionAttribute; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.s3.S3AsyncClient; +import software.amazon.awssdk.services.s3.model.GetObjectResponse; +import software.amazon.awssdk.services.s3.presignedurl.model.PresignedUrlDownloadRequest; + +/** + * WireMock test verifying checksum validation behavior for presigned URL downloads. + */ +@WireMockTest +class PresignedUrlChecksumValidationWiremockTest { + + private static final String BODY = "test-content-for-checksum"; + private static final String CORRECT_CRC32 = "HUUjuQ=="; // CRC32 of "test-content-for-checksum" + private static final String INCORRECT_CRC32 = "AAAAAAAA=="; + + private S3AsyncClient s3Client; + private final AtomicReference checksumValidationStatus = new AtomicReference<>(); + + @BeforeEach + void setUp(WireMockRuntimeInfo wmInfo) { + checksumValidationStatus.set(null); + s3Client = S3AsyncClient.builder() + .endpointOverride(java.net.URI.create(wmInfo.getHttpBaseUrl())) + .region(Region.US_EAST_1) + .credentialsProvider(AnonymousCredentialsProvider.create()) + .forcePathStyle(true) + .overrideConfiguration(c -> c.addExecutionInterceptor(new ExecutionInterceptor() { + @Override + public void afterExecution(Context.AfterExecution context, ExecutionAttributes attrs) { + checksumValidationStatus.set( + attrs.getAttribute(SdkExecutionAttribute.HTTP_RESPONSE_CHECKSUM_VALIDATION)); + } + })) + .build(); + } + + @AfterEach + void tearDown() { + if (s3Client != null) { + s3Client.close(); + } + } + + @Test + void getObject_withMatchingChecksum_shouldSucceed(WireMockRuntimeInfo wmInfo) throws Exception { + stubFor(get(urlPathEqualTo("/test-key")) + .willReturn(aResponse() + .withStatus(200) + .withHeader("Content-Length", String.valueOf(BODY.length())) + .withHeader("x-amz-checksum-crc32", CORRECT_CRC32) + .withHeader("x-amz-checksum-type", "FULL_OBJECT") + .withBody(BODY))); + + URL presignedUrl = new URL(wmInfo.getHttpBaseUrl() + "/test-key?" + + "X-Amz-Algorithm=AWS4-HMAC-SHA256&" + + "X-Amz-SignedHeaders=host%3Bx-amz-checksum-mode&" + + "X-Amz-Signature=fake&" + + "X-Amz-Expires=600"); + + ResponseBytes result = s3Client.presignedUrlExtension() + .getObject( + PresignedUrlDownloadRequest.builder().presignedUrl(presignedUrl).build(), + AsyncResponseTransformer.toBytes()) + .join(); + + assertThat(result.asUtf8String()).isEqualTo(BODY); + assertThat(checksumValidationStatus.get()) + .isEqualTo(ChecksumValidation.VALIDATED); + } + + @Test + void getObject_withMismatchingChecksum_shouldThrow(WireMockRuntimeInfo wmInfo) throws Exception { + stubFor(get(urlPathEqualTo("/test-key")) + .willReturn(aResponse() + .withStatus(200) + .withHeader("Content-Length", String.valueOf(BODY.length())) + .withHeader("x-amz-checksum-crc32", INCORRECT_CRC32) + .withHeader("x-amz-checksum-type", "FULL_OBJECT") + .withBody(BODY))); + + URL presignedUrl = new URL(wmInfo.getHttpBaseUrl() + "/test-key?" + + "X-Amz-Algorithm=AWS4-HMAC-SHA256&" + + "X-Amz-SignedHeaders=host%3Bx-amz-checksum-mode&" + + "X-Amz-Signature=fake&" + + "X-Amz-Expires=600"); + + CompletableFuture> future = s3Client.presignedUrlExtension() + .getObject( + PresignedUrlDownloadRequest.builder().presignedUrl(presignedUrl).build(), + AsyncResponseTransformer.toBytes()); + + assertThatThrownBy(future::join) + .hasCauseInstanceOf(SdkClientException.class) + .hasMessageContaining("checksum"); + } + + @Test + void getObject_withNoChecksumHeader_shouldSucceedWithoutValidation(WireMockRuntimeInfo wmInfo) throws Exception { + stubFor(get(urlPathEqualTo("/test-key")) + .willReturn(aResponse() + .withStatus(200) + .withHeader("Content-Length", String.valueOf(BODY.length())) + .withBody(BODY))); + + URL presignedUrl = new URL(wmInfo.getHttpBaseUrl() + "/test-key?" + + "X-Amz-Algorithm=AWS4-HMAC-SHA256&" + + "X-Amz-SignedHeaders=host&" + + "X-Amz-Signature=fake&" + + "X-Amz-Expires=600"); + + ResponseBytes result = s3Client.presignedUrlExtension() + .getObject( + PresignedUrlDownloadRequest.builder().presignedUrl(presignedUrl).build(), + AsyncResponseTransformer.toBytes()) + .join(); + + assertThat(result.asUtf8String()).isEqualTo(BODY); + assertThat(checksumValidationStatus.get()).isNull(); + } + + @Test + void getObject_withChecksumModeButNoChecksumInResponse_shouldSetAlgorithmNotFound(WireMockRuntimeInfo wmInfo) + throws Exception { + // URL has checksum mode signed, but S3 response has no checksum header (e.g., ranged GET on single-PUT) + stubFor(get(urlPathEqualTo("/test-key")) + .willReturn(aResponse() + .withStatus(200) + .withHeader("Content-Length", String.valueOf(BODY.length())) + .withBody(BODY))); + + URL presignedUrl = new URL(wmInfo.getHttpBaseUrl() + "/test-key?" + + "X-Amz-Algorithm=AWS4-HMAC-SHA256&" + + "X-Amz-SignedHeaders=host%3Bx-amz-checksum-mode&" + + "X-Amz-Signature=fake&" + + "X-Amz-Expires=600"); + + ResponseBytes result = s3Client.presignedUrlExtension() + .getObject( + PresignedUrlDownloadRequest.builder().presignedUrl(presignedUrl).build(), + AsyncResponseTransformer.toBytes()) + .join(); + + assertThat(result.asUtf8String()).isEqualTo(BODY); + assertThat(checksumValidationStatus.get()) + .isEqualTo(ChecksumValidation.CHECKSUM_ALGORITHM_NOT_FOUND); + } + + @Test + void getObject_withCompositeChecksum_shouldSkipValidationAndSucceed(WireMockRuntimeInfo wmInfo) throws Exception { + stubFor(get(urlPathEqualTo("/test-key")) + .willReturn(aResponse() + .withStatus(200) + .withHeader("Content-Length", String.valueOf(BODY.length())) + .withHeader("x-amz-checksum-crc32", "i0T6cA==-3") + .withHeader("x-amz-checksum-type", "COMPOSITE") + .withBody(BODY))); + + URL presignedUrl = new URL(wmInfo.getHttpBaseUrl() + "/test-key?" + + "X-Amz-Algorithm=AWS4-HMAC-SHA256&" + + "X-Amz-SignedHeaders=host%3Bx-amz-checksum-mode&" + + "X-Amz-Signature=fake&" + + "X-Amz-Expires=600"); + + ResponseBytes result = s3Client.presignedUrlExtension() + .getObject( + PresignedUrlDownloadRequest.builder().presignedUrl(presignedUrl).build(), + AsyncResponseTransformer.toBytes()) + .join(); + + assertThat(result.asUtf8String()).isEqualTo(BODY); + assertThat(checksumValidationStatus.get()) + .isEqualTo(ChecksumValidation.FORCE_SKIP); + + } +} diff --git a/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/presignedurl/PresignedUrlDownloadRequestMarshallerTest.java b/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/presignedurl/PresignedUrlDownloadRequestMarshallerTest.java index b89794a2a59c..04c047cb4cfa 100644 --- a/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/presignedurl/PresignedUrlDownloadRequestMarshallerTest.java +++ b/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/presignedurl/PresignedUrlDownloadRequestMarshallerTest.java @@ -158,6 +158,69 @@ void marshall_withNullRequest_shouldThrowException() { .hasMessageContaining("presignedUrlDownloadRequestWrapper must not be null"); } + @Test + void marshall_withChecksumModeInSignedHeaders_shouldAddChecksumHeader() throws Exception { + URL urlWithChecksum = new URL("https://test-bucket.s3.us-east-1.amazonaws.com/test-key?" + + "X-Amz-Algorithm=AWS4-HMAC-SHA256&" + + "X-Amz-SignedHeaders=host%3Bx-amz-checksum-mode&" + + "X-Amz-Signature=abc123&" + + "X-Amz-Expires=600"); + + SdkHttpFullRequest baseRequest = SdkHttpFullRequest.builder() + .method(SdkHttpMethod.GET) + .protocol("https") + .host("example.com") + .build(); + when(mockProtocolMarshaller.marshall(any(PresignedUrlDownloadRequestWrapper.class))) + .thenReturn(baseRequest); + + PresignedUrlDownloadRequestWrapper request = PresignedUrlDownloadRequestWrapper.builder() + .url(urlWithChecksum) + .build(); + SdkHttpFullRequest result = marshaller.marshall(request); + + assertThat(result.headers()).containsKey("x-amz-checksum-mode"); + assertThat(result.firstMatchingHeader("x-amz-checksum-mode")).hasValue("ENABLED"); + } + + @Test + void marshall_withoutChecksumModeInSignedHeaders_shouldNotAddChecksumHeader() throws Exception { + SdkHttpFullRequest baseRequest = SdkHttpFullRequest.builder() + .method(SdkHttpMethod.GET) + .protocol("https") + .host("example.com") + .build(); + when(mockProtocolMarshaller.marshall(any(PresignedUrlDownloadRequestWrapper.class))) + .thenReturn(baseRequest); + + PresignedUrlDownloadRequestWrapper request = PresignedUrlDownloadRequestWrapper.builder() + .url(testUrl) + .build(); + SdkHttpFullRequest result = marshaller.marshall(request); + + assertThat(result.headers()).doesNotContainKey("x-amz-checksum-mode"); + } + + @Test + void marshall_withNoQueryString_shouldNotAddChecksumHeader() throws Exception { + URL urlNoQuery = new URL("https://test-bucket.s3.us-east-1.amazonaws.com/test-key"); + + SdkHttpFullRequest baseRequest = SdkHttpFullRequest.builder() + .method(SdkHttpMethod.GET) + .protocol("https") + .host("example.com") + .build(); + when(mockProtocolMarshaller.marshall(any(PresignedUrlDownloadRequestWrapper.class))) + .thenReturn(baseRequest); + + PresignedUrlDownloadRequestWrapper request = PresignedUrlDownloadRequestWrapper.builder() + .url(urlNoQuery) + .build(); + SdkHttpFullRequest result = marshaller.marshall(request); + + assertThat(result.headers()).doesNotContainKey("x-amz-checksum-mode"); + } + @Test void marshall_withMalformedUrl_shouldThrowSdkClientException() throws Exception { // Setup the mock marshaller to return a properly configured request From 237a2e4c81f3e610b60fcae6063424bf4e6bc156 Mon Sep 17 00:00:00 2001 From: jencymaryjoseph <35571282+jencymaryjoseph@users.noreply.github.com> Date: Tue, 23 Jun 2026 11:06:52 -0700 Subject: [PATCH 25/30] Deprecate IS_DISCOVERED_ENDPOINT, add SKIP_ENDPOINT_RESOLUTION attribute (#7061) --- .../poet/rules/EndpointResolverInterceptorSpec.java | 3 +-- .../poet/rules/RequestEndpointInterceptorSpec.java | 4 ++-- .../rules/AwsEndpointProviderUtils.java.resource | 13 +++++++++++++ .../rules/endpoint-resolve-interceptor-preSra.java | 2 +- ...resolve-interceptor-with-endpointsbasedauth.java | 2 +- ...nt-resolve-interceptor-with-multiauthsigv4a.java | 2 +- ...dpoint-resolve-interceptor-with-stringarray.java | 2 +- .../poet/rules/endpoint-resolve-interceptor.java | 2 +- .../rules/request-set-endpoint-interceptor.java | 2 +- .../interceptor/SdkInternalExecutionAttribute.java | 12 ++++++++++++ .../core/internal/handler/BaseClientHandler.java | 2 ++ .../DefaultAsyncPresignedUrlExtension.java | 3 +-- .../AwsEndpointProviderUtilsTest.java | 13 +++++++++++++ 13 files changed, 50 insertions(+), 12 deletions(-) diff --git a/codegen/src/main/java/software/amazon/awssdk/codegen/poet/rules/EndpointResolverInterceptorSpec.java b/codegen/src/main/java/software/amazon/awssdk/codegen/poet/rules/EndpointResolverInterceptorSpec.java index 57d861ebb025..30123a469fdd 100644 --- a/codegen/src/main/java/software/amazon/awssdk/codegen/poet/rules/EndpointResolverInterceptorSpec.java +++ b/codegen/src/main/java/software/amazon/awssdk/codegen/poet/rules/EndpointResolverInterceptorSpec.java @@ -172,8 +172,7 @@ private MethodSpec modifyRequestMethod(String endpointAuthSchemeStrategyFieldNam String providerVar = "provider"; b.addStatement("$T result = context.request()", SdkRequest.class); - // We skip resolution if the source of the endpoint is the endpoint discovery call - b.beginControlFlow("if ($1T.endpointIsDiscovered(executionAttributes))", + b.beginControlFlow("if ($1T.skipEndpointResolution(executionAttributes))", endpointRulesSpecUtils.rulesRuntimeClassName("AwsEndpointProviderUtils")); b.addStatement("return result"); b.endControlFlow(); diff --git a/codegen/src/main/java/software/amazon/awssdk/codegen/poet/rules/RequestEndpointInterceptorSpec.java b/codegen/src/main/java/software/amazon/awssdk/codegen/poet/rules/RequestEndpointInterceptorSpec.java index 63b5133f180e..c817afddded2 100644 --- a/codegen/src/main/java/software/amazon/awssdk/codegen/poet/rules/RequestEndpointInterceptorSpec.java +++ b/codegen/src/main/java/software/amazon/awssdk/codegen/poet/rules/RequestEndpointInterceptorSpec.java @@ -64,8 +64,8 @@ private MethodSpec modifyHttpRequestMethod() { .addParameter(ExecutionAttributes.class, "executionAttributes"); - // We skip setting the endpoint here if the source of the endpoint is the endpoint discovery call - b.beginControlFlow("if ($1T.endpointIsDiscovered(executionAttributes))", + // We skip setting the endpoint here if endpoint resolution should be skipped + b.beginControlFlow("if ($1T.skipEndpointResolution(executionAttributes))", endpointRulesSpecUtils.rulesRuntimeClassName("AwsEndpointProviderUtils")) .addStatement("return context.httpRequest()") .endControlFlow().build(); diff --git a/codegen/src/main/resources/software/amazon/awssdk/codegen/rules/AwsEndpointProviderUtils.java.resource b/codegen/src/main/resources/software/amazon/awssdk/codegen/rules/AwsEndpointProviderUtils.java.resource index 1d0fedb9f7ff..6da6c020b9bb 100644 --- a/codegen/src/main/resources/software/amazon/awssdk/codegen/rules/AwsEndpointProviderUtils.java.resource +++ b/codegen/src/main/resources/software/amazon/awssdk/codegen/rules/AwsEndpointProviderUtils.java.resource @@ -66,11 +66,24 @@ public final class AwsEndpointProviderUtils { /** * True if the the {@link SdkInternalExecutionAttribute#IS_DISCOVERED_ENDPOINT} attribute is present and its value * is {@code true}, {@code false} otherwise. + * + * @deprecated Use {@link #skipEndpointResolution(ExecutionAttributes)} instead. */ + @Deprecated public static boolean endpointIsDiscovered(ExecutionAttributes attrs) { return attrs.getOptionalAttribute(SdkInternalExecutionAttribute.IS_DISCOVERED_ENDPOINT).orElse(false); } + /** + * True if endpoint resolution should be skipped for the current request. This returns {@code true} if either + * {@link SdkInternalExecutionAttribute#SKIP_ENDPOINT_RESOLUTION} or + * {@link SdkInternalExecutionAttribute#IS_DISCOVERED_ENDPOINT} is set to {@code true}. + */ + public static boolean skipEndpointResolution(ExecutionAttributes attrs) { + return attrs.getOptionalAttribute(SdkInternalExecutionAttribute.SKIP_ENDPOINT_RESOLUTION).orElse(false) + || attrs.getOptionalAttribute(SdkInternalExecutionAttribute.IS_DISCOVERED_ENDPOINT).orElse(false); + } + /** * True if the the {@link SdkInternalExecutionAttribute#DISABLE_HOST_PREFIX_INJECTION} attribute is present and its * value is {@code true}, {@code false} otherwise. diff --git a/codegen/src/test/resources/software/amazon/awssdk/codegen/poet/rules/endpoint-resolve-interceptor-preSra.java b/codegen/src/test/resources/software/amazon/awssdk/codegen/poet/rules/endpoint-resolve-interceptor-preSra.java index e40fb151f74f..de8be3aa52a1 100644 --- a/codegen/src/test/resources/software/amazon/awssdk/codegen/poet/rules/endpoint-resolve-interceptor-preSra.java +++ b/codegen/src/test/resources/software/amazon/awssdk/codegen/poet/rules/endpoint-resolve-interceptor-preSra.java @@ -62,7 +62,7 @@ public QueryResolveEndpointInterceptor() { @Override public SdkRequest modifyRequest(Context.ModifyRequest context, ExecutionAttributes executionAttributes) { SdkRequest result = context.request(); - if (AwsEndpointProviderUtils.endpointIsDiscovered(executionAttributes)) { + if (AwsEndpointProviderUtils.skipEndpointResolution(executionAttributes)) { return result; } QueryEndpointProvider provider = (QueryEndpointProvider) executionAttributes diff --git a/codegen/src/test/resources/software/amazon/awssdk/codegen/poet/rules/endpoint-resolve-interceptor-with-endpointsbasedauth.java b/codegen/src/test/resources/software/amazon/awssdk/codegen/poet/rules/endpoint-resolve-interceptor-with-endpointsbasedauth.java index 116ad513318d..71cd3af7868b 100644 --- a/codegen/src/test/resources/software/amazon/awssdk/codegen/poet/rules/endpoint-resolve-interceptor-with-endpointsbasedauth.java +++ b/codegen/src/test/resources/software/amazon/awssdk/codegen/poet/rules/endpoint-resolve-interceptor-with-endpointsbasedauth.java @@ -50,7 +50,7 @@ public final class QueryResolveEndpointInterceptor implements ExecutionIntercept @Override public SdkRequest modifyRequest(Context.ModifyRequest context, ExecutionAttributes executionAttributes) { SdkRequest result = context.request(); - if (AwsEndpointProviderUtils.endpointIsDiscovered(executionAttributes)) { + if (AwsEndpointProviderUtils.skipEndpointResolution(executionAttributes)) { return result; } QueryEndpointProvider provider = (QueryEndpointProvider) executionAttributes diff --git a/codegen/src/test/resources/software/amazon/awssdk/codegen/poet/rules/endpoint-resolve-interceptor-with-multiauthsigv4a.java b/codegen/src/test/resources/software/amazon/awssdk/codegen/poet/rules/endpoint-resolve-interceptor-with-multiauthsigv4a.java index dd81bd471965..26e225ab6fc5 100644 --- a/codegen/src/test/resources/software/amazon/awssdk/codegen/poet/rules/endpoint-resolve-interceptor-with-multiauthsigv4a.java +++ b/codegen/src/test/resources/software/amazon/awssdk/codegen/poet/rules/endpoint-resolve-interceptor-with-multiauthsigv4a.java @@ -39,7 +39,7 @@ public final class DatabaseResolveEndpointInterceptor implements ExecutionInterc @Override public SdkRequest modifyRequest(Context.ModifyRequest context, ExecutionAttributes executionAttributes) { SdkRequest result = context.request(); - if (AwsEndpointProviderUtils.endpointIsDiscovered(executionAttributes)) { + if (AwsEndpointProviderUtils.skipEndpointResolution(executionAttributes)) { return result; } DatabaseEndpointProvider provider = (DatabaseEndpointProvider) executionAttributes diff --git a/codegen/src/test/resources/software/amazon/awssdk/codegen/poet/rules/endpoint-resolve-interceptor-with-stringarray.java b/codegen/src/test/resources/software/amazon/awssdk/codegen/poet/rules/endpoint-resolve-interceptor-with-stringarray.java index fc8471b3bb4b..23ba5b360972 100644 --- a/codegen/src/test/resources/software/amazon/awssdk/codegen/poet/rules/endpoint-resolve-interceptor-with-stringarray.java +++ b/codegen/src/test/resources/software/amazon/awssdk/codegen/poet/rules/endpoint-resolve-interceptor-with-stringarray.java @@ -41,7 +41,7 @@ public final class SampleSvcResolveEndpointInterceptor implements ExecutionInter @Override public SdkRequest modifyRequest(Context.ModifyRequest context, ExecutionAttributes executionAttributes) { SdkRequest result = context.request(); - if (AwsEndpointProviderUtils.endpointIsDiscovered(executionAttributes)) { + if (AwsEndpointProviderUtils.skipEndpointResolution(executionAttributes)) { return result; } SampleSvcEndpointProvider provider = (SampleSvcEndpointProvider) executionAttributes diff --git a/codegen/src/test/resources/software/amazon/awssdk/codegen/poet/rules/endpoint-resolve-interceptor.java b/codegen/src/test/resources/software/amazon/awssdk/codegen/poet/rules/endpoint-resolve-interceptor.java index 547cbaca685f..1d170e6413e8 100644 --- a/codegen/src/test/resources/software/amazon/awssdk/codegen/poet/rules/endpoint-resolve-interceptor.java +++ b/codegen/src/test/resources/software/amazon/awssdk/codegen/poet/rules/endpoint-resolve-interceptor.java @@ -50,7 +50,7 @@ public final class QueryResolveEndpointInterceptor implements ExecutionIntercept @Override public SdkRequest modifyRequest(Context.ModifyRequest context, ExecutionAttributes executionAttributes) { SdkRequest result = context.request(); - if (AwsEndpointProviderUtils.endpointIsDiscovered(executionAttributes)) { + if (AwsEndpointProviderUtils.skipEndpointResolution(executionAttributes)) { return result; } QueryEndpointProvider provider = (QueryEndpointProvider) executionAttributes diff --git a/codegen/src/test/resources/software/amazon/awssdk/codegen/poet/rules/request-set-endpoint-interceptor.java b/codegen/src/test/resources/software/amazon/awssdk/codegen/poet/rules/request-set-endpoint-interceptor.java index 4cfeedb59e46..8895321a6863 100644 --- a/codegen/src/test/resources/software/amazon/awssdk/codegen/poet/rules/request-set-endpoint-interceptor.java +++ b/codegen/src/test/resources/software/amazon/awssdk/codegen/poet/rules/request-set-endpoint-interceptor.java @@ -14,7 +14,7 @@ public final class QueryRequestSetEndpointInterceptor implements ExecutionInterceptor { @Override public SdkHttpRequest modifyHttpRequest(Context.ModifyHttpRequest context, ExecutionAttributes executionAttributes) { - if (AwsEndpointProviderUtils.endpointIsDiscovered(executionAttributes)) { + if (AwsEndpointProviderUtils.skipEndpointResolution(executionAttributes)) { return context.httpRequest(); } Endpoint endpoint = executionAttributes.getAttribute(SdkInternalExecutionAttribute.RESOLVED_ENDPOINT); diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/interceptor/SdkInternalExecutionAttribute.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/interceptor/SdkInternalExecutionAttribute.java index 529e129e18cd..ede315cbc114 100644 --- a/core/sdk-core/src/main/java/software/amazon/awssdk/core/interceptor/SdkInternalExecutionAttribute.java +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/interceptor/SdkInternalExecutionAttribute.java @@ -125,10 +125,22 @@ public final class SdkInternalExecutionAttribute extends SdkExecutionAttribute { /** * Whether the endpoint on the request is the result of Endpoint Discovery. + * + * @deprecated This attribute is specific to the endpoint discovery feature and should not be used to skip endpoint + * resolution; use {@link #SKIP_ENDPOINT_RESOLUTION} instead. */ + @Deprecated public static final ExecutionAttribute IS_DISCOVERED_ENDPOINT = new ExecutionAttribute<>("IsDiscoveredEndpoint"); + /** + * Whether endpoint resolution should be skipped for the current request. When set to {@code true}, the SDK will not + * invoke the endpoint provider and will use the endpoint already set on the request. This is used for pre-signed URL + * operations and other scenarios where the endpoint is already fully resolved. + */ + public static final ExecutionAttribute SKIP_ENDPOINT_RESOLUTION = + new ExecutionAttribute<>("SkipEndpointResolution"); + /** * The nano time that the current API call attempt began. */ diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/handler/BaseClientHandler.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/handler/BaseClientHandler.java index 9aea1f328993..05a099841374 100644 --- a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/handler/BaseClientHandler.java +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/handler/BaseClientHandler.java @@ -96,6 +96,8 @@ private static SdkHttpFullRequest modifyEndpointHostIfNeeded(SdkHttpFullRequest ClientExecutionParams executionParams) { if (executionParams.discoveredEndpoint() != null) { URI discoveredEndpoint = executionParams.discoveredEndpoint(); + executionParams.putExecutionAttribute(SdkInternalExecutionAttribute.SKIP_ENDPOINT_RESOLUTION, true); + // Keep deprecated attribute for cross-module compatibility with older generated service clients executionParams.putExecutionAttribute(SdkInternalExecutionAttribute.IS_DISCOVERED_ENDPOINT, true); return originalRequest.toBuilder().host(discoveredEndpoint.getHost()).port(discoveredEndpoint.getPort()).build(); } diff --git a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/presignedurl/DefaultAsyncPresignedUrlExtension.java b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/presignedurl/DefaultAsyncPresignedUrlExtension.java index d2a7088aa865..1fcefcd98b9a 100644 --- a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/presignedurl/DefaultAsyncPresignedUrlExtension.java +++ b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/presignedurl/DefaultAsyncPresignedUrlExtension.java @@ -135,8 +135,7 @@ public CompletableFuture getObject( .withRequestConfiguration(clientConfiguration) .withInput(internalRequest) .withMetricCollector(apiCallMetricCollector) - // TODO: Deprecate IS_DISCOVERED_ENDPOINT, use new SKIP_ENDPOINT_RESOLUTION for better semantics - .putExecutionAttribute(SdkInternalExecutionAttribute.IS_DISCOVERED_ENDPOINT, true) + .putExecutionAttribute(SdkInternalExecutionAttribute.SKIP_ENDPOINT_RESOLUTION, true) .putExecutionAttribute(SdkInternalExecutionAttribute.HTTP_CHECKSUM, checksumConfigForUrl(presignedUrlDownloadRequest.presignedUrl())) .withMarshaller(new PresignedUrlDownloadRequestMarshaller(protocolFactory)), diff --git a/test/codegen-generated-classes-test/src/test/java/software/amazon/awssdk/services/endpointproviders/AwsEndpointProviderUtilsTest.java b/test/codegen-generated-classes-test/src/test/java/software/amazon/awssdk/services/endpointproviders/AwsEndpointProviderUtilsTest.java index 83d9d1d0fb59..12c068b2542b 100644 --- a/test/codegen-generated-classes-test/src/test/java/software/amazon/awssdk/services/endpointproviders/AwsEndpointProviderUtilsTest.java +++ b/test/codegen-generated-classes-test/src/test/java/software/amazon/awssdk/services/endpointproviders/AwsEndpointProviderUtilsTest.java @@ -66,6 +66,19 @@ public void endpointIsDiscovered_attrIsTrue_returnsTrue() { assertThat(AwsEndpointProviderUtils.endpointIsDiscovered(attrs)).isTrue(); } + @Test + public void skipEndpointResolution_skipAttrIsTrue_returnsTrue() { + ExecutionAttributes attrs = new ExecutionAttributes(); + attrs.putAttribute(SdkInternalExecutionAttribute.SKIP_ENDPOINT_RESOLUTION, true); + assertThat(AwsEndpointProviderUtils.skipEndpointResolution(attrs)).isTrue(); + } + + @Test + public void skipEndpointResolution_attrsAbsent_returnsFalse() { + ExecutionAttributes attrs = new ExecutionAttributes(); + assertThat(AwsEndpointProviderUtils.skipEndpointResolution(attrs)).isFalse(); + } + @Test public void disableHostPrefixInjection_attrIsFalse_returnsFalse() { ExecutionAttributes attrs = new ExecutionAttributes(); From 8cd9f8cd34c8a7d35327878467f85a0ae43907a5 Mon Sep 17 00:00:00 2001 From: jencymaryjoseph <35571282+jencymaryjoseph@users.noreply.github.com> Date: Tue, 30 Jun 2026 16:37:08 -0700 Subject: [PATCH 26/30] Fix multipart download response metadata for presigned URL and normal paths (#7077) * Add response rewrite infrastructure for multipart download metadata * Fix presigned URL multipart download response metadata and 416 fallback * Fix normal multipart download response metadata * Consolidate normal path integration tests with parameterized tests * Address review: Builder ctor, split(config,mapper) interface method, parameterized tests * Fix transfer manager presigned serial wrapper to use correct progress wrapper * Move responseMapper to SplittingTransformerConfiguration and simplify split wiring * Revert normal object path changes and add presigned custom transformer metadata test * Use single splitWithResponseRewrite * Clarify responseMapper Javadoc on SplittingTransformerConfiguration --- .../SplittingTransformerConfiguration.java | 40 +++++- .../core/async/AsyncResponseTransformer.java | 1 + .../ByteArrayAsyncResponseTransformer.java | 2 +- .../async/ByteArraySplittingTransformer.java | 22 ++- .../internal/async/SplittingTransformer.java | 40 ++++-- .../s3/internal/GenericS3TransferManager.java | 2 +- .../AsyncPresignedUrlExtensionTestSuite.java | 135 ++++++++++++++++++ .../multipart/MultipartDownloadUtils.java | 56 ++++++++ ...ignedUrlMultipartDownloaderSubscriber.java | 4 +- .../multipart/PresignedUrlDownloadHelper.java | 8 +- .../multipart/MultipartDownloadUtilsTest.java | 76 ++++++++++ ...ipartDownloaderSubscriberWiremockTest.java | 118 +++++++++++++++ 12 files changed, 483 insertions(+), 21 deletions(-) diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/SplittingTransformerConfiguration.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/SplittingTransformerConfiguration.java index 766213195203..21b564983c33 100644 --- a/core/sdk-core/src/main/java/software/amazon/awssdk/core/SplittingTransformerConfiguration.java +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/SplittingTransformerConfiguration.java @@ -16,6 +16,7 @@ package software.amazon.awssdk.core; import java.util.Objects; +import java.util.function.UnaryOperator; import software.amazon.awssdk.annotations.SdkPublicApi; import software.amazon.awssdk.core.async.AsyncResponseTransformer; import software.amazon.awssdk.core.internal.async.SplittingTransformer; @@ -35,9 +36,11 @@ public final class SplittingTransformerConfiguration implements ToCopyableBuilde SplittingTransformerConfiguration> { private final Long bufferSizeInBytes; + private final UnaryOperator responseMapper; private SplittingTransformerConfiguration(DefaultBuilder builder) { this.bufferSizeInBytes = Validate.paramNotNull(builder.bufferSize, "bufferSize"); + this.responseMapper = builder.responseMapper; } /** @@ -54,6 +57,14 @@ public Long bufferSizeInBytes() { return bufferSizeInBytes; } + /** + * @return the response mapper applied to the first response before delivery to the upstream transformer, or null if + * not set. See {@link Builder#responseMapper(UnaryOperator)} for semantics. + */ + public UnaryOperator responseMapper() { + return responseMapper; + } + @Override public boolean equals(Object o) { if (this == o) { @@ -65,12 +76,15 @@ public boolean equals(Object o) { SplittingTransformerConfiguration that = (SplittingTransformerConfiguration) o; - return Objects.equals(bufferSizeInBytes, that.bufferSizeInBytes); + return Objects.equals(bufferSizeInBytes, that.bufferSizeInBytes) + && Objects.equals(responseMapper, that.responseMapper); } @Override public int hashCode() { - return bufferSizeInBytes != null ? bufferSizeInBytes.hashCode() : 0; + int result = bufferSizeInBytes != null ? bufferSizeInBytes.hashCode() : 0; + result = 31 * result + (responseMapper != null ? responseMapper.hashCode() : 0); + return result; } @Override @@ -94,13 +108,29 @@ public interface Builder extends CopyableBuilderOnly applied by the default {@code split} implementation. A transformer that overrides {@code split} (such as a + * parallel, file-based one) may not read it, in which case it has no effect. + * + * @param responseMapper a function to transform the response before delivery, or null for no mapping + * @return This object for method chaining. + */ + Builder responseMapper(UnaryOperator responseMapper); } private static final class DefaultBuilder implements Builder { private Long bufferSize; + private UnaryOperator responseMapper; private DefaultBuilder(SplittingTransformerConfiguration configuration) { this.bufferSize = configuration.bufferSizeInBytes; + this.responseMapper = configuration.responseMapper; } private DefaultBuilder() { @@ -112,6 +142,12 @@ public Builder bufferSizeInBytes(Long bufferSize) { return this; } + @Override + public Builder responseMapper(UnaryOperator responseMapper) { + this.responseMapper = responseMapper; + return this; + } + @Override public SplittingTransformerConfiguration build() { return new SplittingTransformerConfiguration(this); diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/async/AsyncResponseTransformer.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/async/AsyncResponseTransformer.java index 4c27501e6340..dd4a5140170f 100644 --- a/core/sdk-core/src/main/java/software/amazon/awssdk/core/async/AsyncResponseTransformer.java +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/async/AsyncResponseTransformer.java @@ -138,6 +138,7 @@ default SplitResult split(SplittingTransformerConfiguration .builder() .upstreamResponseTransformer(this) .maximumBufferSizeInBytes(splitConfig.bufferSizeInBytes()) + .responseMapper(splitConfig.responseMapper()) .resultFuture(future) .build(); return AsyncResponseTransformer.SplitResult.builder() diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/async/ByteArrayAsyncResponseTransformer.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/async/ByteArrayAsyncResponseTransformer.java index e250eab650b0..3c94080f604c 100644 --- a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/async/ByteArrayAsyncResponseTransformer.java +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/async/ByteArrayAsyncResponseTransformer.java @@ -72,7 +72,7 @@ public void exceptionOccurred(Throwable throwable) { public SplitResult> split(SplittingTransformerConfiguration splitConfig) { CompletableFuture> future = new CompletableFuture<>(); SdkPublisher> transformer = - new ByteArraySplittingTransformer<>(this, future); + new ByteArraySplittingTransformer<>(this, future, splitConfig.responseMapper()); return AsyncResponseTransformer.SplitResult.>builder() .publisher(transformer) .resultFuture(future) diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/async/ByteArraySplittingTransformer.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/async/ByteArraySplittingTransformer.java index 2531f7f8166b..fa5761d01bd2 100644 --- a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/async/ByteArraySplittingTransformer.java +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/async/ByteArraySplittingTransformer.java @@ -23,10 +23,12 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReference; +import java.util.function.UnaryOperator; import org.reactivestreams.Subscriber; import org.reactivestreams.Subscription; import software.amazon.awssdk.annotations.SdkInternalApi; import software.amazon.awssdk.core.ResponseBytes; +import software.amazon.awssdk.core.SdkResponse; import software.amazon.awssdk.core.async.AsyncResponseTransformer; import software.amazon.awssdk.core.async.SdkPublisher; import software.amazon.awssdk.core.exception.SdkClientException; @@ -84,12 +86,30 @@ public class ByteArraySplittingTransformer implements SdkPublisher buffers; + private final UnaryOperator responseMapper; + public ByteArraySplittingTransformer(AsyncResponseTransformer> upstreamResponseTransformer, CompletableFuture> resultFuture) { + this(upstreamResponseTransformer, resultFuture, UnaryOperator.identity()); + } + + public ByteArraySplittingTransformer(AsyncResponseTransformer> + upstreamResponseTransformer, + CompletableFuture> resultFuture, + UnaryOperator responseMapper) { this.upstreamResponseTransformer = upstreamResponseTransformer; this.resultFuture = resultFuture; this.buffers = new ConcurrentHashMap<>(); + this.responseMapper = responseMapper != null ? responseMapper : UnaryOperator.identity(); + } + + @SuppressWarnings("unchecked") + private ResponseT mapResponse(ResponseT response) { + if (!(response instanceof SdkResponse)) { + return response; + } + return (ResponseT) responseMapper.apply((SdkResponse) response); } @Override @@ -181,7 +201,7 @@ private void handleSubscriptionCancel() { CompletableFuture> upstreamPrepareFuture = upstreamResponseTransformer.prepare(); CompletableFutureUtils.forwardResultTo(upstreamPrepareFuture, resultFuture); - upstreamResponseTransformer.onResponse(responseT.get()); + upstreamResponseTransformer.onResponse(mapResponse(responseT.get())); int totalPartCount = nextPartNumber.get() - 1; if (buffers.size() != totalPartCount) { diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/async/SplittingTransformer.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/async/SplittingTransformer.java index d4cf1c7a2356..0b94dd2f7c34 100644 --- a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/async/SplittingTransformer.java +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/async/SplittingTransformer.java @@ -19,9 +19,11 @@ import java.util.concurrent.CompletableFuture; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; +import java.util.function.UnaryOperator; import org.reactivestreams.Subscriber; import org.reactivestreams.Subscription; import software.amazon.awssdk.annotations.SdkInternalApi; +import software.amazon.awssdk.core.SdkResponse; import software.amazon.awssdk.core.SplittingTransformerConfiguration; import software.amazon.awssdk.core.async.AsyncResponseTransformer; import software.amazon.awssdk.core.async.SdkPublisher; @@ -112,16 +114,18 @@ public class SplittingTransformer implements SdkPublisher upstreamResponseTransformer, - Long maximumBufferSizeInBytes, - CompletableFuture resultFuture) { + private final UnaryOperator responseMapper; + + private SplittingTransformer(Builder builder) { this.upstreamResponseTransformer = Validate.paramNotNull( - upstreamResponseTransformer, "upstreamResponseTransformer"); - this.resultFuture = Validate.paramNotNull( - resultFuture, "resultFuture"); - Validate.notNull(maximumBufferSizeInBytes, "maximumBufferSizeInBytes"); + builder.upstreamResponseTransformer, "upstreamResponseTransformer"); + this.resultFuture = Validate.paramNotNull(builder.returnFuture, "resultFuture"); + Validate.notNull(builder.maximumBufferSize, "maximumBufferSizeInBytes"); this.maximumBufferInBytes = Validate.isPositive( - maximumBufferSizeInBytes, "maximumBufferSizeInBytes"); + builder.maximumBufferSize, "maximumBufferSizeInBytes"); + this.responseMapper = builder.responseMapper != null + ? builder.responseMapper + : UnaryOperator.identity(); this.resultFuture.whenComplete((r, e) -> { if (e == null) { @@ -133,6 +137,14 @@ private SplittingTransformer(AsyncResponseTransformer upstre }); } + @SuppressWarnings("unchecked") + private ResponseT mapResponse(ResponseT response) { + if (!(response instanceof SdkResponse)) { + return response; + } + return (ResponseT) responseMapper.apply((SdkResponse) response); + } + /** * @param downstreamSubscriber the {@link Subscriber} to the individual AsyncResponseTransformer */ @@ -296,7 +308,7 @@ public CompletableFuture prepare() { public void onResponse(ResponseT response) { if (onResponseCalled.compareAndSet(false, true)) { log.trace(() -> "calling onResponse on the upstream transformer"); - upstreamResponseTransformer.onResponse(response); + upstreamResponseTransformer.onResponse(mapResponse(response)); } this.response = response; } @@ -393,6 +405,7 @@ public static final class Builder { private Long maximumBufferSize; private CompletableFuture returnFuture; private AsyncResponseTransformer upstreamResponseTransformer; + private UnaryOperator responseMapper; private Builder() { } @@ -437,10 +450,13 @@ public Builder resultFuture(CompletableFuture retur return this; } + public Builder responseMapper(UnaryOperator responseMapper) { + this.responseMapper = responseMapper; + return this; + } + public SplittingTransformer build() { - return new SplittingTransformer<>(this.upstreamResponseTransformer, - this.maximumBufferSize, - this.returnFuture); + return new SplittingTransformer<>(this); } } } diff --git a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/GenericS3TransferManager.java b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/GenericS3TransferManager.java index c86a3fea12fb..f92c5196a095 100644 --- a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/GenericS3TransferManager.java +++ b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/GenericS3TransferManager.java @@ -334,7 +334,7 @@ private CopyObjectRequest attachSdkAttribute(CopyObjectRequest copyObjectRequest } private GetObjectRequest attachSdkAttribute(GetObjectRequest request, - Consumer builderMutation) { + Consumer builderMutation) { AwsRequestOverrideConfiguration modifiedRequestOverrideConfig = request.overrideConfiguration() .map(o -> o.toBuilder().applyMutation(builderMutation).build()) diff --git a/services/s3/src/it/java/software/amazon/awssdk/services/s3/presignedurl/AsyncPresignedUrlExtensionTestSuite.java b/services/s3/src/it/java/software/amazon/awssdk/services/s3/presignedurl/AsyncPresignedUrlExtensionTestSuite.java index 1ff737b52850..b5a3edbfd8f7 100644 --- a/services/s3/src/it/java/software/amazon/awssdk/services/s3/presignedurl/AsyncPresignedUrlExtensionTestSuite.java +++ b/services/s3/src/it/java/software/amazon/awssdk/services/s3/presignedurl/AsyncPresignedUrlExtensionTestSuite.java @@ -24,7 +24,9 @@ import java.nio.file.Path; import java.time.Duration; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; +import java.util.Random; import java.util.UUID; import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeUnit; @@ -41,15 +43,21 @@ import software.amazon.awssdk.core.ResponseBytes; import software.amazon.awssdk.core.async.AsyncRequestBody; import software.amazon.awssdk.core.async.AsyncResponseTransformer; +import software.amazon.awssdk.core.sync.RequestBody; import software.amazon.awssdk.metrics.MetricCollection; import software.amazon.awssdk.metrics.MetricPublisher; import software.amazon.awssdk.services.s3.S3AsyncClient; import software.amazon.awssdk.services.s3.S3IntegrationTestBase; +import software.amazon.awssdk.services.s3.model.ChecksumAlgorithm; import software.amazon.awssdk.services.s3.model.ChecksumMode; +import software.amazon.awssdk.services.s3.model.CompletedMultipartUpload; +import software.amazon.awssdk.services.s3.model.CompletedPart; +import software.amazon.awssdk.services.s3.model.CreateMultipartUploadResponse; import software.amazon.awssdk.services.s3.model.DeleteObjectRequest; import software.amazon.awssdk.services.s3.model.GetObjectResponse; import software.amazon.awssdk.services.s3.model.PutObjectRequest; import software.amazon.awssdk.services.s3.model.S3Exception; +import software.amazon.awssdk.services.s3.model.UploadPartResponse; import software.amazon.awssdk.services.s3.presigner.S3Presigner; import software.amazon.awssdk.services.s3.presigner.model.PresignedGetObjectRequest; import software.amazon.awssdk.services.s3.presignedurl.model.PresignedUrlDownloadRequest; @@ -70,8 +78,10 @@ public abstract class AsyncPresignedUrlExtensionTestSuite extends S3IntegrationT protected static String testGetObjectKey; protected static String testLargeObjectKey; + protected static String testMpuChecksumKey; protected static String testObjectContent; protected static byte[] testLargeObjectContent; + protected static byte[] testMpuObjectContent; protected static String expectedLargeObjectMd5; protected abstract S3AsyncClient createS3AsyncClient(); @@ -105,6 +115,7 @@ static void setUpTestSuite() throws Exception { .build(), AsyncRequestBody.fromBytes(testLargeObjectContent) ).join(); + uploadMpuObjectWithChecksum(); S3TestUtils.addCleanupTask(AsyncPresignedUrlExtensionTestSuite.class, () -> { s3.deleteObject(DeleteObjectRequest.builder() .bucket(testBucket) @@ -114,6 +125,10 @@ static void setUpTestSuite() throws Exception { .bucket(testBucket) .key(testLargeObjectKey) .build()); + s3.deleteObject(DeleteObjectRequest.builder() + .bucket(testBucket) + .key(testMpuChecksumKey) + .build()); deleteBucketAndAllContents(testBucket); }); } @@ -152,6 +167,8 @@ void getObject_withValidPresignedUrl_returnsContent(String testDescription, assertThat(response.asByteArray().length).isEqualTo(testLargeObjectContent.length); } assertThat(response.response()).isNotNull(); + assertThat(response.response().contentLength()).isEqualTo((long) testLargeObjectContent.length); + assertThat(response.response().sdkHttpResponse().firstMatchingHeader("x-amz-request-id")).isPresent(); } } @@ -239,6 +256,8 @@ public void close() {} assertThat(response).isNotNull(); assertThat(downloadFile).exists(); assertThat(downloadFile.toFile().length()).isEqualTo(testLargeObjectContent.length); + assertThat(response.contentLength()).isEqualTo((long) testLargeObjectContent.length); + assertThat(response.sdkHttpResponse().firstMatchingHeader("x-amz-request-id")).isPresent(); assertThat(collectedMetrics).isNotEmpty(); } } @@ -359,7 +378,123 @@ static Stream rangeTestData() { ); } + @Test + void getObject_withRangeRequest_preservesPartialMetadata() throws Exception { + PresignedUrlDownloadRequest request = createRequestForKey(testLargeObjectKey, "bytes=0-1048575"); + ResponseBytes response = + presignedUrlExtension.getObject(request, AsyncResponseTransformer.toBytes()) + .get(30, TimeUnit.SECONDS); + + assertThat(response.response().contentLength()).isEqualTo(1048576L); + assertThat(response.response().contentRange()).isNotNull(); + assertThat(response.response().sdkHttpResponse().firstMatchingHeader("x-amz-request-id")).isPresent(); + } + + @ParameterizedTest(name = "getObject_largeObject_{0}_hasCorrectFullObjectMetadata") + @MethodSource("transformerTypes") + void getObject_largeObject_hasCorrectFullObjectMetadata(String type) throws Exception { + PresignedUrlDownloadRequest request = createRequestForKey(testLargeObjectKey); + + if ("toFile".equals(type)) { + Path downloadFile = temporaryFolder.resolve("large-metadata-test-" + UUID.randomUUID() + ".bin"); + GetObjectResponse response = + presignedUrlExtension.getObject(request, downloadFile) + .get(60, TimeUnit.SECONDS); + + assertThat(response.contentLength()).isEqualTo((long) testLargeObjectContent.length); + assertThat(downloadFile.toFile().length()).isEqualTo(testLargeObjectContent.length); + assertThat(response.sdkHttpResponse().firstMatchingHeader("x-amz-request-id")).isPresent(); + } else { + ResponseBytes response = + presignedUrlExtension.getObject(request, AsyncResponseTransformer.toBytes()) + .get(60, TimeUnit.SECONDS); + + assertThat(response.asByteArray().length).isEqualTo(testLargeObjectContent.length); + assertThat(response.response().contentLength()).isEqualTo((long) testLargeObjectContent.length); + assertThat(response.response().sdkHttpResponse().firstMatchingHeader("x-amz-request-id")).isPresent(); + } + } + + static Stream transformerTypes() { + return Stream.of("toFile", "toBytes"); + } + + @ParameterizedTest(name = "getObject_mpuObject_{0}_hasCorrectMetadata") + @MethodSource("checksumModes") + void getObject_mpuObject_hasCorrectMetadata(String mode) throws Exception { + PresignedUrlDownloadRequest request; + if ("withChecksumMode".equals(mode)) { + PresignedGetObjectRequest presigned = presigner.presignGetObject(r -> r + .getObjectRequest(req -> req.bucket(testBucket).key(testMpuChecksumKey) + .checksumMode(ChecksumMode.ENABLED)) + .signatureDuration(Duration.ofMinutes(10))); + request = PresignedUrlDownloadRequest.builder().presignedUrl(presigned.url()).build(); + } else { + request = PresignedUrlDownloadRequest.builder() + .presignedUrl(createPresignedUrl(testMpuChecksumKey)) + .build(); + } + + ResponseBytes response = + presignedUrlExtension.getObject(request, AsyncResponseTransformer.toBytes()) + .get(60, TimeUnit.SECONDS); + + assertThat(response.asByteArray().length).isEqualTo(testMpuObjectContent.length); + assertThat(response.response().contentLength()).isEqualTo((long) testMpuObjectContent.length); + assertThat(response.response().sdkHttpResponse().firstMatchingHeader("x-amz-request-id")).isPresent(); + } + + static Stream checksumModes() { + return Stream.of("withChecksumMode", "withoutChecksumMode"); + } + + @Test + void getObject_mpuObjectWithRange_preservesPartialMetadata() throws Exception { + PresignedUrlDownloadRequest request = PresignedUrlDownloadRequest.builder() + .presignedUrl(createPresignedUrl(testMpuChecksumKey)) + .range("bytes=0-1048575") + .build(); + + ResponseBytes response = + presignedUrlExtension.getObject(request, AsyncResponseTransformer.toBytes()) + .get(30, TimeUnit.SECONDS); + + assertThat(response.response().contentLength()).isEqualTo(1048576L); + assertThat(response.response().contentRange()).contains("bytes 0-1048575/"); + assertThat(response.response().sdkHttpResponse().firstMatchingHeader("x-amz-request-id")).isPresent(); + } + // Helper methods + private static void uploadMpuObjectWithChecksum() { + testMpuChecksumKey = generateRandomObjectKey() + "-mpu-checksum"; + int partSize = 5 * 1024 * 1024; + int numParts = 2; + testMpuObjectContent = new byte[partSize * numParts]; + new Random(42).nextBytes(testMpuObjectContent); + + CreateMultipartUploadResponse createResp = + s3.createMultipartUpload(b -> b.bucket(testBucket).key(testMpuChecksumKey) + .checksumAlgorithm(ChecksumAlgorithm.CRC32)); + String uploadId = createResp.uploadId(); + List parts = new ArrayList<>(); + + for (int i = 0; i < numParts; i++) { + byte[] partData = Arrays.copyOfRange(testMpuObjectContent, i * partSize, (i + 1) * partSize); + final int partNum = i + 1; + UploadPartResponse uploadResp = s3.uploadPart( + b -> b.bucket(testBucket).key(testMpuChecksumKey).uploadId(uploadId).partNumber(partNum) + .checksumAlgorithm(ChecksumAlgorithm.CRC32), + RequestBody.fromBytes(partData)); + parts.add(CompletedPart.builder() + .partNumber(partNum).eTag(uploadResp.eTag()) + .checksumCRC32(uploadResp.checksumCRC32()).build()); + } + + s3.completeMultipartUpload(b -> b.bucket(testBucket).key(testMpuChecksumKey).uploadId(uploadId) + .multipartUpload(CompletedMultipartUpload.builder() + .parts(parts).build())); + } + private static String generateRandomObjectKey() { return "async-presigned-url-extension-test-" + UUID.randomUUID(); } diff --git a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/MultipartDownloadUtils.java b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/MultipartDownloadUtils.java index c7e9885e34d3..a9145f1bc1dc 100644 --- a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/MultipartDownloadUtils.java +++ b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/MultipartDownloadUtils.java @@ -23,7 +23,13 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; import software.amazon.awssdk.annotations.SdkInternalApi; +import software.amazon.awssdk.core.SdkResponse; +import software.amazon.awssdk.core.SplittingTransformerConfiguration; +import software.amazon.awssdk.core.async.AsyncResponseTransformer; +import software.amazon.awssdk.core.async.AsyncResponseTransformer.SplitResult; +import software.amazon.awssdk.services.s3.model.ChecksumType; import software.amazon.awssdk.services.s3.model.GetObjectRequest; +import software.amazon.awssdk.services.s3.model.GetObjectResponse; import software.amazon.awssdk.services.s3.model.S3Request; @SdkInternalApi @@ -126,4 +132,54 @@ public static long calculateTotalParts(long contentLength, long partSize) { } + /** + * Rewrites a first-part response to represent the full object. + * + * @param firstPartResponse the GetObjectResponse from the first part request + * @return full-object response with total content-length, full content-range, + * and checksum values nulled if checksum type is COMPOSITE + */ + public static GetObjectResponse toFullObjectResponse(GetObjectResponse firstPartResponse) { + String contentRange = firstPartResponse.contentRange(); + Optional totalOpt = parseContentRangeForTotalSize(contentRange); + if (!totalOpt.isPresent()) { + return firstPartResponse; + } + long totalLength = totalOpt.get(); + String fullRange = "bytes 0-" + (totalLength - 1) + "/" + totalLength; + + GetObjectResponse.Builder builder = firstPartResponse.toBuilder() + .contentLength(totalLength) + .contentRange(fullRange); + + if (firstPartResponse.checksumType() == ChecksumType.COMPOSITE) { + builder.sdkFields().stream() + .filter(f -> f.memberName().startsWith("Checksum") && !"ChecksumType".equals(f.memberName())) + .forEach(f -> f.set(builder, null)); + } + + return builder.build(); + } + + /** + * Splits the given transformer with a response mapper that applies {@link #toFullObjectResponse} + * to the first part's response before it reaches the customer's transformer. + */ + public static SplitResult splitWithResponseRewrite( + AsyncResponseTransformer transformer, + SplittingTransformerConfiguration splitConfig) { + SplittingTransformerConfiguration configWithMapper = + splitConfig.toBuilder() + .responseMapper(MultipartDownloadUtils::mapToFullObjectResponse) + .build(); + return transformer.split(configWithMapper); + } + + private static SdkResponse mapToFullObjectResponse(SdkResponse response) { + if (response instanceof GetObjectResponse) { + return toFullObjectResponse((GetObjectResponse) response); + } + return response; + } + } diff --git a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/ParallelPresignedUrlMultipartDownloaderSubscriber.java b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/ParallelPresignedUrlMultipartDownloaderSubscriber.java index 88031de5c733..3497aa3745e6 100644 --- a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/ParallelPresignedUrlMultipartDownloaderSubscriber.java +++ b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/ParallelPresignedUrlMultipartDownloaderSubscriber.java @@ -191,7 +191,7 @@ private void sendFirstRequest(AsyncResponseTransformer CompletableFuture downloadObject( doMultipartDownload(presignedRequest, asyncResponseTransformer) .whenComplete((result, error) -> { Throwable cause = error instanceof CompletionException ? error.getCause() : error; - if (cause instanceof EmptyObjectRangeNotSatisfiableException) { + // Parallel path wraps it as EmptyObjectRangeNotSatisfiableException; + // serial path (toBytes, custom transformers) surfaces raw S3Exception. + if (cause instanceof EmptyObjectRangeNotSatisfiableException + || isRangeNotSatisfiable(cause)) { log.debug(() -> "Received 416 on first request, falling back to non-range GET for empty object"); asyncPresignedUrlExtension.getObject(presignedRequest, asyncResponseTransformer) .whenComplete((r, e) -> { @@ -93,8 +96,9 @@ private CompletableFuture doMultipartDownload( SplittingTransformerConfiguration splittingConfig = SplittingTransformerConfiguration.builder() .bufferSizeInBytes(bufferSizeInBytes) .build(); + AsyncResponseTransformer.SplitResult split = - asyncResponseTransformer.split(splittingConfig); + MultipartDownloadUtils.splitWithResponseRewrite(asyncResponseTransformer, splittingConfig); if (split.parallelSplitSupported()) { return downloadPartsInParallel(presignedRequest, split); diff --git a/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/multipart/MultipartDownloadUtilsTest.java b/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/multipart/MultipartDownloadUtilsTest.java index acc535b686eb..b88c6cd8a33d 100644 --- a/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/multipart/MultipartDownloadUtilsTest.java +++ b/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/multipart/MultipartDownloadUtilsTest.java @@ -19,7 +19,10 @@ import static software.amazon.awssdk.services.s3.multipart.S3MultipartExecutionAttribute.MULTIPART_DOWNLOAD_RESUME_CONTEXT; import org.junit.jupiter.api.Test; +import software.amazon.awssdk.http.SdkHttpResponse; +import software.amazon.awssdk.services.s3.model.ChecksumType; import software.amazon.awssdk.services.s3.model.GetObjectRequest; +import software.amazon.awssdk.services.s3.model.GetObjectResponse; class MultipartDownloadUtilsTest { @@ -99,4 +102,77 @@ void calculateTotalParts_shouldCalculateCorrectly() { assertThat(MultipartDownloadUtils.calculateTotalParts(Long.MAX_VALUE, 2)) .isEqualTo((Long.MAX_VALUE / 2) + 1); } + + @Test + void toFullObjectResponse_setsContentLengthAndContentRange() { + GetObjectResponse response = GetObjectResponse.builder() + .contentLength(1024L) + .contentRange("bytes 0-1023/4096") + .eTag("\"abc123\"") + .build(); + response = setHttpResponse(response, 206, "Partial Content"); + + GetObjectResponse result = MultipartDownloadUtils.toFullObjectResponse(response); + + assertThat(result.contentLength()).isEqualTo(4096L); + assertThat(result.contentRange()).isEqualTo("bytes 0-4095/4096"); + assertThat(result.eTag()).isEqualTo("\"abc123\""); + } + + @Test + void toFullObjectResponse_noContentRange_returnsUnchanged() { + GetObjectResponse response = GetObjectResponse.builder() + .contentLength(1024L) + .build(); + response = setHttpResponse(response, 206, "Partial Content"); + + GetObjectResponse result = MultipartDownloadUtils.toFullObjectResponse(response); + + assertThat(result.contentLength()).isEqualTo(1024L); + } + + @Test + void toFullObjectResponse_compositeChecksum_nullsChecksumValues() { + GetObjectResponse response = GetObjectResponse.builder() + .contentLength(8388608L) + .contentRange("bytes 0-8388607/25165824") + .checksumType(ChecksumType.COMPOSITE) + .checksumCRC32("/g/pWA==") + .build(); + response = setHttpResponse(response, 206, "Partial Content"); + + GetObjectResponse result = MultipartDownloadUtils.toFullObjectResponse(response); + + assertThat(result.checksumType()).isEqualTo(ChecksumType.COMPOSITE); + assertThat(result.checksumCRC32()).isNull(); + assertThat(result.checksumCRC32C()).isNull(); + assertThat(result.checksumCRC64NVME()).isNull(); + assertThat(result.checksumSHA1()).isNull(); + assertThat(result.checksumSHA256()).isNull(); + assertThat(result.contentLength()).isEqualTo(25165824L); + } + + @Test + void toFullObjectResponse_fullObjectChecksum_preservesChecksumValues() { + GetObjectResponse response = GetObjectResponse.builder() + .contentLength(8388608L) + .contentRange("bytes 0-8388607/25165824") + .checksumType(ChecksumType.FULL_OBJECT) + .checksumCRC32("abc123") + .build(); + response = setHttpResponse(response, 206, "Partial Content"); + + GetObjectResponse result = MultipartDownloadUtils.toFullObjectResponse(response); + + assertThat(result.checksumType()).isEqualTo(ChecksumType.FULL_OBJECT); + assertThat(result.checksumCRC32()).isEqualTo("abc123"); + } + + private static GetObjectResponse setHttpResponse(GetObjectResponse response, int statusCode, String statusText) { + SdkHttpResponse httpResponse = SdkHttpResponse.builder() + .statusCode(statusCode) + .statusText(statusText) + .build(); + return (GetObjectResponse) response.toBuilder().sdkHttpResponse(httpResponse).build(); + } } diff --git a/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/multipart/PresignedUrlMultipartDownloaderSubscriberWiremockTest.java b/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/multipart/PresignedUrlMultipartDownloaderSubscriberWiremockTest.java index c71534842bb9..ad05af389452 100644 --- a/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/multipart/PresignedUrlMultipartDownloaderSubscriberWiremockTest.java +++ b/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/multipart/PresignedUrlMultipartDownloaderSubscriberWiremockTest.java @@ -31,6 +31,7 @@ import java.net.MalformedURLException; import java.net.URI; import java.net.URL; +import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; @@ -44,8 +45,11 @@ import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; +import org.reactivestreams.Subscriber; +import org.reactivestreams.Subscription; import software.amazon.awssdk.core.ResponseBytes; import software.amazon.awssdk.core.async.AsyncResponseTransformer; +import software.amazon.awssdk.core.async.SdkPublisher; import software.amazon.awssdk.core.exception.SdkClientException; import software.amazon.awssdk.services.s3.S3AsyncClient; import software.amazon.awssdk.services.s3.model.GetObjectResponse; @@ -71,6 +75,9 @@ public void setup(WireMockRuntimeInfo wiremock) throws MalformedURLException { .build(); s3AsyncClient = S3AsyncClient.builder() .endpointOverride(URI.create("http://localhost:" + wiremock.getHttpPort())) + .credentialsProvider(software.amazon.awssdk.auth.credentials.StaticCredentialsProvider.create( + software.amazon.awssdk.auth.credentials.AwsBasicCredentials.create("accessKey", "secretKey"))) + .region(software.amazon.awssdk.regions.Region.US_EAST_1) .multipartEnabled(true) .multipartConfiguration(multipartConfig) .build(); @@ -339,6 +346,59 @@ void presignedUrlDownload_withRangeHeader_emptyObject_shouldThrow416(String tran .hasRootCauseInstanceOf(S3Exception.class); } + /** + * Verifies that custom serial transformers on the presigned URL path correctly trigger + * the empty-object 416 fallback. + */ + @Test + void presignedUrlDownload_emptyObject_customTransformer_fallbackWorks() { + // Range request → 416 (simulates empty object race) + stubFor(get(urlEqualTo(PRESIGNED_URL_PATH)) + .withHeader("Range", matching("bytes=.*")) + .willReturn(aResponse() + .withStatus(416) + .withBody("InvalidRange" + + "The requested range is not satisfiable"))); + + // Non-range fallback GET → 200 with empty body (the correct fallback for empty object) + stubFor(get(urlEqualTo(PRESIGNED_URL_PATH)) + .withHeader("Range", absent()) + .willReturn(aResponse() + .withStatus(200) + .withHeader("Content-Length", "0") + .withBody(""))); + + AsyncResponseTransformer customTransformer = + new AsyncResponseTransformer() { + private CompletableFuture future; + @Override + public CompletableFuture prepare() { + future = new CompletableFuture<>(); + return future; + } + @Override public void onResponse(GetObjectResponse r) { } + @Override public void onStream(SdkPublisher p) { + p.subscribe(new Subscriber() { + @Override public void onSubscribe(Subscription s) { s.request(Long.MAX_VALUE); } + @Override public void onNext(ByteBuffer b) { } + @Override public void onError(Throwable t) { future.completeExceptionally(t); } + @Override public void onComplete() { future.complete("done"); } + }); + } + @Override public void exceptionOccurred(Throwable e) { future.completeExceptionally(e); } + }; + + PresignedUrlDownloadRequest request = PresignedUrlDownloadRequest.builder() + .presignedUrl(presignedUrl) + .build(); + + // Should succeed: 416 triggers fallback → non-range GET returns 200 + String result = s3AsyncClient.presignedUrlExtension() + .getObject(request, customTransformer) + .join(); + assertThat(result).isEqualTo("done"); + } + @AfterEach void cleanup() { if (tempFile != null && Files.exists(tempFile)) { @@ -486,4 +546,62 @@ private void stubResponseWithContentLengthMismatch() { .withHeader("ETag", "\"test-etag\"") .withBody(Arrays.copyOfRange(TEST_DATA, 0, 8)))); } + + @Test + void presignedUrlDownload_customTransformer_hasFullObjectMetadata() { + // Stub two parts: 16 bytes each, total 32 + stubFor(get(urlEqualTo(PRESIGNED_URL_PATH)) + .withHeader("Range", matching("bytes=0-15")) + .willReturn(aResponse() + .withStatus(206) + .withHeader("Content-Length", "16") + .withHeader("Content-Range", "bytes 0-15/32") + .withHeader("ETag", "\"test-etag\"") + .withBody(Arrays.copyOfRange(TEST_DATA, 0, 16)))); + + stubFor(get(urlEqualTo(PRESIGNED_URL_PATH)) + .withHeader("Range", matching("bytes=16-31")) + .willReturn(aResponse() + .withStatus(206) + .withHeader("Content-Length", "16") + .withHeader("Content-Range", "bytes 16-31/32") + .withHeader("ETag", "\"test-etag\"") + .withBody(Arrays.copyOfRange(TEST_DATA, 16, 32)))); + + AsyncResponseTransformer customTransformer = + new AsyncResponseTransformer() { + private CompletableFuture future; + private GetObjectResponse response; + + @Override + public CompletableFuture prepare() { + future = new CompletableFuture<>(); + return future; + } + @Override public void onResponse(GetObjectResponse r) { this.response = r; } + @Override public void onStream(SdkPublisher p) { + p.subscribe(new Subscriber() { + @Override public void onSubscribe(Subscription s) { s.request(Long.MAX_VALUE); } + @Override public void onNext(ByteBuffer b) { } + @Override public void onError(Throwable t) { future.completeExceptionally(t); } + @Override public void onComplete() { + future.complete("contentLength=" + response.contentLength() + + "|contentRange=" + response.contentRange()); + } + }); + } + @Override public void exceptionOccurred(Throwable e) { future.completeExceptionally(e); } + }; + + PresignedUrlDownloadRequest request = PresignedUrlDownloadRequest.builder() + .presignedUrl(presignedUrl) + .build(); + + String result = s3AsyncClient.presignedUrlExtension() + .getObject(request, customTransformer) + .join(); + + assertThat(result).contains("contentLength=32"); + assertThat(result).contains("contentRange=bytes 0-31/32"); + } } From 2255143599a7ca3548ff48b64fd60a69de4911dd Mon Sep 17 00:00:00 2001 From: jencymaryjoseph <35571282+jencymaryjoseph@users.noreply.github.com> Date: Tue, 7 Jul 2026 14:39:46 -0700 Subject: [PATCH 27/30] Merge branch 'master' into feature/master/pre-signed-url-getobject --- .../poet/client/test-presignedurl-async.java | 79 ++++++++++++++++++- 1 file changed, 75 insertions(+), 4 deletions(-) diff --git a/codegen/src/test/resources/software/amazon/awssdk/codegen/poet/client/test-presignedurl-async.java b/codegen/src/test/resources/software/amazon/awssdk/codegen/poet/client/test-presignedurl-async.java index dcef20b7814b..824155b1d9e5 100644 --- a/codegen/src/test/resources/software/amazon/awssdk/codegen/poet/client/test-presignedurl-async.java +++ b/codegen/src/test/resources/software/amazon/awssdk/codegen/poet/client/test-presignedurl-async.java @@ -6,13 +6,18 @@ import java.util.List; import java.util.Optional; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionException; import java.util.function.Consumer; import java.util.function.Function; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import software.amazon.awssdk.annotations.Generated; import software.amazon.awssdk.annotations.SdkInternalApi; +import software.amazon.awssdk.awscore.client.config.AwsClientOption; import software.amazon.awssdk.awscore.client.handler.AwsAsyncClientHandler; +import software.amazon.awssdk.awscore.endpoints.AwsEndpointAttribute; +import software.amazon.awssdk.awscore.endpoints.AwsEndpointProviderUtils; +import software.amazon.awssdk.awscore.endpoints.authscheme.EndpointAuthScheme; import software.amazon.awssdk.awscore.exception.AwsServiceException; import software.amazon.awssdk.awscore.internal.AwsProtocolMetadata; import software.amazon.awssdk.awscore.internal.AwsServiceProtocol; @@ -20,14 +25,20 @@ import software.amazon.awssdk.core.RequestOverrideConfiguration; import software.amazon.awssdk.core.SdkPlugin; import software.amazon.awssdk.core.SdkRequest; +import software.amazon.awssdk.core.SelectedAuthScheme; import software.amazon.awssdk.core.client.config.ClientOverrideConfiguration; import software.amazon.awssdk.core.client.config.SdkClientConfiguration; import software.amazon.awssdk.core.client.config.SdkClientOption; import software.amazon.awssdk.core.client.handler.AsyncClientHandler; import software.amazon.awssdk.core.client.handler.ClientExecutionParams; +import software.amazon.awssdk.core.exception.SdkClientException; import software.amazon.awssdk.core.http.HttpResponseHandler; +import software.amazon.awssdk.core.interceptor.ExecutionAttributes; +import software.amazon.awssdk.core.interceptor.SdkInternalExecutionAttribute; import software.amazon.awssdk.core.metrics.CoreMetric; import software.amazon.awssdk.core.retry.RetryMode; +import software.amazon.awssdk.endpoints.Endpoint; +import software.amazon.awssdk.http.auth.spi.scheme.AuthSchemeOption; import software.amazon.awssdk.metrics.MetricCollector; import software.amazon.awssdk.metrics.MetricPublisher; import software.amazon.awssdk.metrics.NoOpMetricCollector; @@ -37,6 +48,11 @@ import software.amazon.awssdk.protocols.json.BaseAwsJsonProtocolFactory; import software.amazon.awssdk.protocols.json.JsonOperationMetadata; import software.amazon.awssdk.retries.api.RetryStrategy; +import software.amazon.awssdk.services.json.auth.scheme.JsonAuthSchemeParams; +import software.amazon.awssdk.services.json.auth.scheme.JsonAuthSchemeProvider; +import software.amazon.awssdk.services.json.endpoints.JsonEndpointParams; +import software.amazon.awssdk.services.json.endpoints.JsonEndpointProvider; +import software.amazon.awssdk.services.json.endpoints.internal.JsonEndpointResolverUtils; import software.amazon.awssdk.services.json.internal.JsonServiceClientConfigurationBuilder; import software.amazon.awssdk.services.json.internal.ServiceVersionInfo; import software.amazon.awssdk.services.json.internal.presignedurl.DefaultAsyncPresignedUrlExtension; @@ -47,6 +63,7 @@ import software.amazon.awssdk.services.json.presignedurl.AsyncPresignedUrlExtension; import software.amazon.awssdk.services.json.transform.APostOperationRequestMarshaller; import software.amazon.awssdk.utils.CompletableFutureUtils; +import software.amazon.awssdk.utils.Validate; /** * Internal implementation of {@link JsonAsyncClient}. @@ -104,7 +121,7 @@ public CompletableFuture aPostOperation(APostOperationRe List metricPublishers = resolveMetricPublishers(clientConfiguration, aPostOperationRequest .overrideConfiguration().orElse(null)); MetricCollector apiCallMetricCollector = metricPublishers.isEmpty() ? NoOpMetricCollector.create() : MetricCollector - .create("ApiCall"); + .create("ApiCall"); try { apiCallMetricCollector.reportMetric(CoreMetric.SERVICE_ID, "Json Service"); apiCallMetricCollector.reportMetric(CoreMetric.OPERATION_NAME, "APostOperation"); @@ -130,10 +147,16 @@ public CompletableFuture aPostOperation(APostOperationRe CompletableFuture executeFuture = clientHandler .execute(new ClientExecutionParams() - .withOperationName("APostOperation").withProtocolMetadata(protocolMetadata) + .withOperationName("APostOperation") + .withProtocolMetadata(protocolMetadata) .withMarshaller(new APostOperationRequestMarshaller(protocolFactory)) - .withResponseHandler(responseHandler).withErrorResponseHandler(errorResponseHandler) - .withRequestConfiguration(clientConfiguration).withMetricCollector(apiCallMetricCollector) + .withResponseHandler(responseHandler) + .withErrorResponseHandler(errorResponseHandler) + .withRequestConfiguration(clientConfiguration) + .withMetricCollector(apiCallMetricCollector) + .withAuthSchemeOptionsResolver( + r -> resolveAuthSchemeOptions(r, "APostOperation", clientConfiguration)) + .withEndpointResolver((r, a) -> resolveEndpoint(r, a, "APostOperation")) .withInput(aPostOperationRequest)); CompletableFuture whenCompleted = executeFuture.whenComplete((r, e) -> { metricPublishers.forEach(p -> p.publish(apiCallMetricCollector.collect())); @@ -181,6 +204,54 @@ private static List resolveMetricPublishers(SdkClientConfigurat return publishers; } + private List resolveAuthSchemeOptions(SdkRequest request, String operationName, + SdkClientConfiguration clientConfiguration) { + JsonAuthSchemeProvider requestAuthSchemeProvider = request + .overrideConfiguration() + .flatMap(c -> c.authSchemeProvider()) + .map(p -> Validate + .isInstanceOf(JsonAuthSchemeProvider.class, p, "Expected an instance of JsonAuthSchemeProvider")) + .orElse(null); + JsonAuthSchemeProvider authSchemeProvider = requestAuthSchemeProvider != null ? requestAuthSchemeProvider : Validate + .isInstanceOf(JsonAuthSchemeProvider.class, clientConfiguration.option(SdkClientOption.AUTH_SCHEME_PROVIDER), + "Expected an instance of JsonAuthSchemeProvider"); + JsonAuthSchemeParams.Builder paramsBuilder = JsonAuthSchemeParams.builder().operation(operationName); + paramsBuilder.region(clientConfiguration.option(AwsClientOption.AWS_REGION)); + List options = authSchemeProvider.resolveAuthScheme(paramsBuilder.build()); + return options; + } + + private Endpoint resolveEndpoint(SdkRequest request, ExecutionAttributes executionAttributes, String operationName) { + JsonEndpointProvider provider = (JsonEndpointProvider) executionAttributes + .getAttribute(SdkInternalExecutionAttribute.ENDPOINT_PROVIDER); + try { + JsonEndpointParams endpointParams = JsonEndpointResolverUtils.ruleParams(request, executionAttributes); + Endpoint endpoint = provider.resolveEndpoint(endpointParams).join(); + if (!AwsEndpointProviderUtils.disableHostPrefixInjection(executionAttributes)) { + Optional hostPrefix = JsonEndpointResolverUtils.hostPrefix(operationName, request); + if (hostPrefix.isPresent()) { + endpoint = AwsEndpointProviderUtils.addHostPrefix(endpoint, hostPrefix.get()); + } + } + List endpointAuthSchemes = endpoint.attribute(AwsEndpointAttribute.AUTH_SCHEMES); + SelectedAuthScheme selectedAuthScheme = executionAttributes + .getAttribute(SdkInternalExecutionAttribute.SELECTED_AUTH_SCHEME); + if (endpointAuthSchemes != null && selectedAuthScheme != null) { + selectedAuthScheme = JsonEndpointResolverUtils.authSchemeWithEndpointSignerProperties(endpointAuthSchemes, + selectedAuthScheme); + executionAttributes.putAttribute(SdkInternalExecutionAttribute.SELECTED_AUTH_SCHEME, selectedAuthScheme); + } + JsonEndpointResolverUtils.setMetricValues(endpoint, executionAttributes); + return endpoint; + } catch (CompletionException e) { + Throwable cause = e.getCause(); + if (cause instanceof SdkClientException) { + throw (SdkClientException) cause; + } + throw SdkClientException.create("Endpoint resolution failed: " + cause.getMessage(), cause); + } + } + private void updateRetryStrategyClientConfiguration(SdkClientConfiguration.Builder configuration) { ClientOverrideConfiguration.Builder builder = configuration.asOverrideConfigurationBuilder(); RetryMode retryMode = builder.retryMode(); From 6ccab72797fd38ab5333b9e1665117f8337636de Mon Sep 17 00:00:00 2001 From: jencymaryjoseph <35571282+jencymaryjoseph@users.noreply.github.com> Date: Wed, 8 Jul 2026 10:07:44 -0700 Subject: [PATCH 28/30] Add changelog entry for presigned URL download feature --- .changes/next-release/feature-AmazonS3-a6f238d.json | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 .changes/next-release/feature-AmazonS3-a6f238d.json diff --git a/.changes/next-release/feature-AmazonS3-a6f238d.json b/.changes/next-release/feature-AmazonS3-a6f238d.json new file mode 100644 index 000000000000..fcbf1d071765 --- /dev/null +++ b/.changes/next-release/feature-AmazonS3-a6f238d.json @@ -0,0 +1,6 @@ +{ + "type": "feature", + "category": "Amazon S3", + "description": "Add presigned URL download support to S3AsyncClient and S3 Transfer Manager. Customers can now download S3 objects using pre-signed URLs through the SDK's async client pipeline without needing AWS credentials configured.", + "contributor": "jencymaryjoseph" +} From 45c92bf43d8340c708529ab02574cbf94fa69290 Mon Sep 17 00:00:00 2001 From: jencymaryjoseph <35571282+jencymaryjoseph@users.noreply.github.com> Date: Wed, 8 Jul 2026 15:07:48 -0700 Subject: [PATCH 29/30] Remove contributor name --- .changes/next-release/feature-AmazonS3-a6f238d.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.changes/next-release/feature-AmazonS3-a6f238d.json b/.changes/next-release/feature-AmazonS3-a6f238d.json index fcbf1d071765..51666e41d5c5 100644 --- a/.changes/next-release/feature-AmazonS3-a6f238d.json +++ b/.changes/next-release/feature-AmazonS3-a6f238d.json @@ -2,5 +2,5 @@ "type": "feature", "category": "Amazon S3", "description": "Add presigned URL download support to S3AsyncClient and S3 Transfer Manager. Customers can now download S3 objects using pre-signed URLs through the SDK's async client pipeline without needing AWS credentials configured.", - "contributor": "jencymaryjoseph" + "contributor": "" } From 8aede75a25d4ff0e35426fabb28cd17b5cbff052 Mon Sep 17 00:00:00 2001 From: jencymaryjoseph <35571282+jencymaryjoseph@users.noreply.github.com> Date: Fri, 10 Jul 2026 13:27:51 -0700 Subject: [PATCH 30/30] Merge branch 'master' into feature/master/pre-signed-url-getobject --- .../poet/client/test-presignedurl-async.java | 25 +++++++++++-------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/codegen/src/test/resources/software/amazon/awssdk/codegen/poet/client/test-presignedurl-async.java b/codegen/src/test/resources/software/amazon/awssdk/codegen/poet/client/test-presignedurl-async.java index 824155b1d9e5..5e9fdee4bcc6 100644 --- a/codegen/src/test/resources/software/amazon/awssdk/codegen/poet/client/test-presignedurl-async.java +++ b/codegen/src/test/resources/software/amazon/awssdk/codegen/poet/client/test-presignedurl-async.java @@ -31,12 +31,14 @@ import software.amazon.awssdk.core.client.config.SdkClientOption; import software.amazon.awssdk.core.client.handler.AsyncClientHandler; import software.amazon.awssdk.core.client.handler.ClientExecutionParams; +import software.amazon.awssdk.core.endpoint.EndpointResolver; import software.amazon.awssdk.core.exception.SdkClientException; import software.amazon.awssdk.core.http.HttpResponseHandler; import software.amazon.awssdk.core.interceptor.ExecutionAttributes; import software.amazon.awssdk.core.interceptor.SdkInternalExecutionAttribute; import software.amazon.awssdk.core.metrics.CoreMetric; import software.amazon.awssdk.core.retry.RetryMode; +import software.amazon.awssdk.core.spi.identity.AuthSchemeOptionsResolver; import software.amazon.awssdk.endpoints.Endpoint; import software.amazon.awssdk.http.auth.spi.scheme.AuthSchemeOption; import software.amazon.awssdk.metrics.MetricCollector; @@ -147,17 +149,12 @@ public CompletableFuture aPostOperation(APostOperationRe CompletableFuture executeFuture = clientHandler .execute(new ClientExecutionParams() - .withOperationName("APostOperation") - .withProtocolMetadata(protocolMetadata) + .withOperationName("APostOperation").withProtocolMetadata(protocolMetadata) .withMarshaller(new APostOperationRequestMarshaller(protocolFactory)) - .withResponseHandler(responseHandler) - .withErrorResponseHandler(errorResponseHandler) - .withRequestConfiguration(clientConfiguration) - .withMetricCollector(apiCallMetricCollector) - .withAuthSchemeOptionsResolver( - r -> resolveAuthSchemeOptions(r, "APostOperation", clientConfiguration)) - .withEndpointResolver((r, a) -> resolveEndpoint(r, a, "APostOperation")) - .withInput(aPostOperationRequest)); + .withResponseHandler(responseHandler).withErrorResponseHandler(errorResponseHandler) + .withRequestConfiguration(clientConfiguration).withMetricCollector(apiCallMetricCollector) + .withAuthSchemeOptionsResolver(authSchemeResolver("APostOperation", clientConfiguration)) + .withEndpointResolver(endpointResolver("APostOperation")).withInput(aPostOperationRequest)); CompletableFuture whenCompleted = executeFuture.whenComplete((r, e) -> { metricPublishers.forEach(p -> p.publish(apiCallMetricCollector.collect())); }); @@ -252,6 +249,14 @@ private Endpoint resolveEndpoint(SdkRequest request, ExecutionAttributes executi } } + private AuthSchemeOptionsResolver authSchemeResolver(String operationName, SdkClientConfiguration clientConfiguration) { + return r -> resolveAuthSchemeOptions(r, operationName, clientConfiguration); + } + + private EndpointResolver endpointResolver(String operationName) { + return (r, a) -> resolveEndpoint(r, a, operationName); + } + private void updateRetryStrategyClientConfiguration(SdkClientConfiguration.Builder configuration) { ClientOverrideConfiguration.Builder builder = configuration.asOverrideConfigurationBuilder(); RetryMode retryMode = builder.retryMode();