diff --git a/.changes/next-release/feature-AmazonS3-a6f238d.json b/.changes/next-release/feature-AmazonS3-a6f238d.json new file mode 100644 index 000000000000..51666e41d5c5 --- /dev/null +++ b/.changes/next-release/feature-AmazonS3-a6f238d.json @@ -0,0 +1,6 @@ +{ + "type": "feature", + "category": "Amazon S3", + "description": "Add presigned URL download support to S3AsyncClient and S3 Transfer Manager. Customers can now download S3 objects using pre-signed URLs through the SDK's async client pipeline without needing AWS credentials configured.", + "contributor": "" +} diff --git a/codegen/src/main/java/software/amazon/awssdk/codegen/AddMetadata.java b/codegen/src/main/java/software/amazon/awssdk/codegen/AddMetadata.java index 41d1f32693f8..f5764111371c 100644 --- a/codegen/src/main/java/software/amazon/awssdk/codegen/AddMetadata.java +++ b/codegen/src/main/java/software/amazon/awssdk/codegen/AddMetadata.java @@ -62,6 +62,7 @@ public static Metadata constructMetadata(ServiceModel serviceModel, .withDocumentation(serviceModel.getDocumentation()) .withServiceAbbreviation(serviceMetadata.getServiceAbbreviation()) .withBatchmanagerPackageName(namingStrategy.getBatchManagerPackageName(serviceName)) + .withPresignedUrlPackageName(namingStrategy.getPresignedUrlPackageName(serviceName)) .withServiceFullName(serviceMetadata.getServiceFullName()) .withServiceName(serviceName) .withSyncClient(String.format(Constant.SYNC_CLIENT_CLASS_NAME_PATTERN, serviceName)) diff --git a/codegen/src/main/java/software/amazon/awssdk/codegen/internal/Constant.java b/codegen/src/main/java/software/amazon/awssdk/codegen/internal/Constant.java index 5384975f49fb..a0bf024211b7 100644 --- a/codegen/src/main/java/software/amazon/awssdk/codegen/internal/Constant.java +++ b/codegen/src/main/java/software/amazon/awssdk/codegen/internal/Constant.java @@ -78,6 +78,8 @@ public final class Constant { public static final String PACKAGE_NAME_BATCHMANAGER_PATTERN = "%s.batchmanager"; + public static final String PACKAGE_NAME_PRESIGNEDURL_PATTERN = "%s.presignedurl"; + public static final String PACKAGE_NAME_CUSTOM_AUTH_PATTERN = "%s.auth"; public static final String AUTH_POLICY_ENUM_CLASS_DIR = "software/amazon/awssdk/auth/policy/actions"; diff --git a/codegen/src/main/java/software/amazon/awssdk/codegen/model/config/customization/CustomizationConfig.java b/codegen/src/main/java/software/amazon/awssdk/codegen/model/config/customization/CustomizationConfig.java index fe5bc8703920..d15ef90cd2e1 100644 --- a/codegen/src/main/java/software/amazon/awssdk/codegen/model/config/customization/CustomizationConfig.java +++ b/codegen/src/main/java/software/amazon/awssdk/codegen/model/config/customization/CustomizationConfig.java @@ -351,6 +351,11 @@ public class CustomizationConfig { */ private boolean batchManagerSupported; + /** + * A boolean flag to indicate if Presigned URL Extension is supported. + */ + private boolean presignedUrlExtensionSupported; + /** * A boolean flag to indicate if the fast unmarshaller code path is enabled. */ @@ -951,6 +956,14 @@ public void setBatchManagerSupported(boolean batchManagerSupported) { this.batchManagerSupported = batchManagerSupported; } + public boolean getPresignedUrlExtensionSupported() { + return presignedUrlExtensionSupported; + } + + public void setPresignedUrlExtensionSupported(boolean presignedUrlExtensionSupported) { + this.presignedUrlExtensionSupported = presignedUrlExtensionSupported; + } + public boolean getEnableFastUnmarshaller() { return enableFastUnmarshaller; } diff --git a/codegen/src/main/java/software/amazon/awssdk/codegen/model/intermediate/Metadata.java b/codegen/src/main/java/software/amazon/awssdk/codegen/model/intermediate/Metadata.java index 4eb10f6bf105..3cd8f5df4113 100644 --- a/codegen/src/main/java/software/amazon/awssdk/codegen/model/intermediate/Metadata.java +++ b/codegen/src/main/java/software/amazon/awssdk/codegen/model/intermediate/Metadata.java @@ -73,6 +73,8 @@ public class Metadata { private String batchManagerPackageName; + private String presignedUrlPackageName; + private String endpointRulesPackageName; private String authSchemePackageName; @@ -815,4 +817,20 @@ public String getFullBatchManagerPackageName() { return joinPackageNames(rootPackageName, getBatchManagerPackageName()); } + public Metadata withPresignedUrlPackageName(String presignedUrlPackageName) { + setPresignedUrlPackageName(presignedUrlPackageName); + return this; + } + + public String getPresignedUrlPackageName() { + return presignedUrlPackageName; + } + + public void setPresignedUrlPackageName(String presignedUrlPackageName) { + this.presignedUrlPackageName = presignedUrlPackageName; + } + + public String getFullPresignedUrlPackageName() { + return joinPackageNames(rootPackageName, getPresignedUrlPackageName()); + } } diff --git a/codegen/src/main/java/software/amazon/awssdk/codegen/naming/DefaultNamingStrategy.java b/codegen/src/main/java/software/amazon/awssdk/codegen/naming/DefaultNamingStrategy.java index b9fcf41e6e0f..dc5257d9c9aa 100644 --- a/codegen/src/main/java/software/amazon/awssdk/codegen/naming/DefaultNamingStrategy.java +++ b/codegen/src/main/java/software/amazon/awssdk/codegen/naming/DefaultNamingStrategy.java @@ -220,6 +220,11 @@ public String getJmesPathPackageName(String serviceName) { public String getBatchManagerPackageName(String serviceName) { return getCustomizedPackageName(concatServiceNameIfShareModel(serviceName), Constant.PACKAGE_NAME_BATCHMANAGER_PATTERN); } + + @Override + public String getPresignedUrlPackageName(String serviceName) { + return getCustomizedPackageName(concatServiceNameIfShareModel(serviceName), Constant.PACKAGE_NAME_PRESIGNEDURL_PATTERN); + } @Override public String getSmokeTestPackageName(String serviceName) { diff --git a/codegen/src/main/java/software/amazon/awssdk/codegen/naming/NamingStrategy.java b/codegen/src/main/java/software/amazon/awssdk/codegen/naming/NamingStrategy.java index 5b22363970f0..1047a3a1904b 100644 --- a/codegen/src/main/java/software/amazon/awssdk/codegen/naming/NamingStrategy.java +++ b/codegen/src/main/java/software/amazon/awssdk/codegen/naming/NamingStrategy.java @@ -93,6 +93,11 @@ public interface NamingStrategy { * Retrieve the batchManager package name that should be used based on the service name. */ String getBatchManagerPackageName(String serviceName); + + /** + * Retrieve the presignedUrl package name that should be used based on the service name. + */ + String getPresignedUrlPackageName(String serviceName); /** * Retrieve the smote test package name that should be used based on the service name. diff --git a/codegen/src/main/java/software/amazon/awssdk/codegen/poet/PoetExtension.java b/codegen/src/main/java/software/amazon/awssdk/codegen/poet/PoetExtension.java index f119507809c2..9e4fea7f6b31 100644 --- a/codegen/src/main/java/software/amazon/awssdk/codegen/poet/PoetExtension.java +++ b/codegen/src/main/java/software/amazon/awssdk/codegen/poet/PoetExtension.java @@ -209,4 +209,8 @@ public ClassName getBatchManagerAsyncInterface() { return ClassName.get(model.getMetadata().getFullBatchManagerPackageName(), model.getMetadata().getServiceName() + "AsyncBatchManager"); } + + public ClassName getPresignedUrlExtensionAsyncInterface() { + return ClassName.get(model.getMetadata().getFullPresignedUrlPackageName(), "AsyncPresignedUrlExtension"); + } } diff --git a/codegen/src/main/java/software/amazon/awssdk/codegen/poet/client/AsyncClientClass.java b/codegen/src/main/java/software/amazon/awssdk/codegen/poet/client/AsyncClientClass.java index 95ed15f3c6f9..045040d12f45 100644 --- a/codegen/src/main/java/software/amazon/awssdk/codegen/poet/client/AsyncClientClass.java +++ b/codegen/src/main/java/software/amazon/awssdk/codegen/poet/client/AsyncClientClass.java @@ -548,6 +548,26 @@ protected void addBatchManagerMethod(Builder type) { type.addMethod(batchManager); } + + @Override + protected void addPresignedUrlExtensionMethod(Builder type) { + ClassName returnType = poetExtensions.getPresignedUrlExtensionAsyncInterface(); + String internalPresignedUrlPackage = model.getMetadata().getFullInternalPackageName() + ".presignedurl"; + ClassName implClass = ClassName.get(internalPresignedUrlPackage, "DefaultAsyncPresignedUrlExtension"); + + MethodSpec presignedUrlExtension = MethodSpec.methodBuilder("presignedUrlExtension") + .addModifiers(PUBLIC) + .addAnnotation(Override.class) + .returns(returnType) + .addStatement("return new $T(clientHandler," + + " protocolFactory, " + + "clientConfiguration," + + " protocolMetadata)", + implClass) + .build(); + + type.addMethod(presignedUrlExtension); + } private MethodSpec resolveMetricPublishersMethod() { String clientConfigName = "clientConfiguration"; diff --git a/codegen/src/main/java/software/amazon/awssdk/codegen/poet/client/AsyncClientInterface.java b/codegen/src/main/java/software/amazon/awssdk/codegen/poet/client/AsyncClientInterface.java index cbd5f3cb3e9a..fe6d40015703 100644 --- a/codegen/src/main/java/software/amazon/awssdk/codegen/poet/client/AsyncClientInterface.java +++ b/codegen/src/main/java/software/amazon/awssdk/codegen/poet/client/AsyncClientInterface.java @@ -99,6 +99,9 @@ public TypeSpec poetSpec() { if (model.getCustomizationConfig().getBatchManagerSupported()) { addBatchManagerMethod(result); } + if (model.getCustomizationConfig().getPresignedUrlExtensionSupported()) { + addPresignedUrlExtensionMethod(result); + } result.addMethod(serviceClientConfigMethod()); addAdditionalMethods(result); addCloseMethod(result); @@ -174,6 +177,16 @@ protected void addBatchManagerMethod(TypeSpec.Builder type) { + "configuration set on this client.", returnType); type.addMethod(batchManagerOperationBody(builder).build()); } + + protected void addPresignedUrlExtensionMethod(TypeSpec.Builder type) { + ClassName returnType = poetExtensions.getPresignedUrlExtensionAsyncInterface(); + MethodSpec.Builder builder = MethodSpec.methodBuilder("presignedUrlExtension") + .addModifiers(PUBLIC) + .returns(returnType) + .addJavadoc("Creates an instance of {@link $T} object with the " + + "configuration set on this client.", returnType); + type.addMethod(presignedUrlExtensionOperationBody(builder).build()); + } @Override public ClassName className() { @@ -550,5 +563,10 @@ protected MethodSpec.Builder batchManagerOperationBody(MethodSpec.Builder builde return builder.addModifiers(DEFAULT, PUBLIC) .addStatement("throw new $T()", UnsupportedOperationException.class); } + + protected MethodSpec.Builder presignedUrlExtensionOperationBody(MethodSpec.Builder builder) { + return builder.addModifiers(DEFAULT, PUBLIC) + .addStatement("throw new $T()", UnsupportedOperationException.class); + } } diff --git a/codegen/src/main/java/software/amazon/awssdk/codegen/poet/client/DelegatingAsyncClientClass.java b/codegen/src/main/java/software/amazon/awssdk/codegen/poet/client/DelegatingAsyncClientClass.java index 5af2c45ad0bc..3fd5fcec9f51 100644 --- a/codegen/src/main/java/software/amazon/awssdk/codegen/poet/client/DelegatingAsyncClientClass.java +++ b/codegen/src/main/java/software/amazon/awssdk/codegen/poet/client/DelegatingAsyncClientClass.java @@ -222,4 +222,9 @@ protected MethodSpec.Builder waiterOperationBody(MethodSpec.Builder builder) { protected MethodSpec.Builder batchManagerOperationBody(MethodSpec.Builder builder) { return builder.addAnnotation(Override.class).addStatement("return delegate.batchManager()"); } + + @Override + protected MethodSpec.Builder presignedUrlExtensionOperationBody(MethodSpec.Builder builder) { + return builder.addAnnotation(Override.class).addStatement("return delegate.presignedUrlExtension()"); + } } diff --git a/codegen/src/test/java/software/amazon/awssdk/codegen/poet/ClientTestModels.java b/codegen/src/test/java/software/amazon/awssdk/codegen/poet/ClientTestModels.java index acd08400564a..7426688dd92d 100644 --- a/codegen/src/test/java/software/amazon/awssdk/codegen/poet/ClientTestModels.java +++ b/codegen/src/test/java/software/amazon/awssdk/codegen/poet/ClientTestModels.java @@ -605,6 +605,18 @@ public static IntermediateModel batchManagerModels() { return new IntermediateModelBuilder(models).build(); } + + public static IntermediateModel presignedUrlExtensionModels() { + File serviceModel = new File(ClientTestModels.class.getResource("client/c2j/presignedurl/service-2.json").getFile()); + File customizationModel = new File(ClientTestModels.class.getResource("client/c2j/presignedurl/customization.config").getFile()); + + C2jModels models = C2jModels.builder() + .serviceModel(getServiceModel(serviceModel)) + .customizationConfig(getCustomizationConfig(customizationModel)) + .build(); + + return new IntermediateModelBuilder(models).build(); + } private static ServiceModel getServiceModel(File file) { return ModelLoaderUtils.loadModel(ServiceModel.class, file); diff --git a/codegen/src/test/java/software/amazon/awssdk/codegen/poet/client/AsyncClientClassTest.java b/codegen/src/test/java/software/amazon/awssdk/codegen/poet/client/AsyncClientClassTest.java index 6e2082fc4175..ab3bfeb68c92 100644 --- a/codegen/src/test/java/software/amazon/awssdk/codegen/poet/client/AsyncClientClassTest.java +++ b/codegen/src/test/java/software/amazon/awssdk/codegen/poet/client/AsyncClientClassTest.java @@ -24,6 +24,7 @@ import static software.amazon.awssdk.codegen.poet.ClientTestModels.customPackageModels; import static software.amazon.awssdk.codegen.poet.ClientTestModels.endpointDiscoveryModels; import static software.amazon.awssdk.codegen.poet.ClientTestModels.opsWithSigv4a; +import static software.amazon.awssdk.codegen.poet.ClientTestModels.presignedUrlExtensionModels; import static software.amazon.awssdk.codegen.poet.ClientTestModels.serviceWithCustomContextParamsModels; import static software.amazon.awssdk.codegen.poet.ClientTestModels.queryServiceModels; import static software.amazon.awssdk.codegen.poet.ClientTestModels.restJsonServiceModels; @@ -102,6 +103,12 @@ public void asyncClientBatchManager() { ClassSpec aSyncClientBatchManager = createAsyncClientClass(batchManagerModels()); assertThat(aSyncClientBatchManager, generatesTo("test-batchmanager-async.java")); } + + @Test + public void asyncClientPresignedUrlExtension() { + ClassSpec asyncClientPresignedUrlExtension = createAsyncClientClass(presignedUrlExtensionModels()); + assertThat(asyncClientPresignedUrlExtension, generatesTo("test-presignedurl-async.java")); + } @Test public void asyncClientWithStreamingUnsignedPayload() { diff --git a/codegen/src/test/java/software/amazon/awssdk/codegen/poet/client/AsyncClientInterfaceTest.java b/codegen/src/test/java/software/amazon/awssdk/codegen/poet/client/AsyncClientInterfaceTest.java index fda4c6f2b459..aae45702e837 100644 --- a/codegen/src/test/java/software/amazon/awssdk/codegen/poet/client/AsyncClientInterfaceTest.java +++ b/codegen/src/test/java/software/amazon/awssdk/codegen/poet/client/AsyncClientInterfaceTest.java @@ -17,6 +17,7 @@ import static org.hamcrest.MatcherAssert.assertThat; import static software.amazon.awssdk.codegen.poet.ClientTestModels.batchManagerModels; +import static software.amazon.awssdk.codegen.poet.ClientTestModels.presignedUrlExtensionModels; import static software.amazon.awssdk.codegen.poet.ClientTestModels.restJsonServiceModels; import static software.amazon.awssdk.codegen.poet.PoetMatchers.generatesTo; @@ -35,4 +36,10 @@ public void asyncClientInterfaceWithBatchManager() { ClassSpec asyncClientInterface = new AsyncClientInterface(batchManagerModels()); assertThat(asyncClientInterface, generatesTo("test-json-async-client-interface-batchmanager.java")); } + + @Test + public void asyncClientInterfaceWithPresignedUrlExtension() { + ClassSpec asyncClientInterface = new AsyncClientInterface(presignedUrlExtensionModels()); + assertThat(asyncClientInterface, generatesTo("test-json-async-client-interface-presignedurl.java")); + } } diff --git a/codegen/src/test/java/software/amazon/awssdk/codegen/poet/client/DelegatingAsyncClientClassTest.java b/codegen/src/test/java/software/amazon/awssdk/codegen/poet/client/DelegatingAsyncClientClassTest.java index a12ab3ac659c..a9af3d6a8b3b 100644 --- a/codegen/src/test/java/software/amazon/awssdk/codegen/poet/client/DelegatingAsyncClientClassTest.java +++ b/codegen/src/test/java/software/amazon/awssdk/codegen/poet/client/DelegatingAsyncClientClassTest.java @@ -16,6 +16,7 @@ package software.amazon.awssdk.codegen.poet.client; import static org.hamcrest.MatcherAssert.assertThat; +import static software.amazon.awssdk.codegen.poet.ClientTestModels.presignedUrlExtensionModels; import static software.amazon.awssdk.codegen.poet.ClientTestModels.restJsonServiceModels; import static software.amazon.awssdk.codegen.poet.PoetMatchers.generatesTo; @@ -28,4 +29,11 @@ public void delegatingAsyncClientClass() { new DelegatingAsyncClientClass(restJsonServiceModels()); assertThat(asyncClientDecoratorAbstractClass, generatesTo("test-abstract-async-client-class.java")); } + + @Test + public void delegatingAsyncClientClassWithPresignedUrlExtension() { + DelegatingAsyncClientClass asyncClientDecoratorAbstractClass = + new DelegatingAsyncClientClass(presignedUrlExtensionModels()); + assertThat(asyncClientDecoratorAbstractClass, generatesTo("test-abstract-async-client-class-presignedurl.java")); + } } diff --git a/codegen/src/test/resources/software/amazon/awssdk/codegen/poet/client/c2j/presignedurl/customization.config b/codegen/src/test/resources/software/amazon/awssdk/codegen/poet/client/c2j/presignedurl/customization.config new file mode 100644 index 000000000000..17d6c2306501 --- /dev/null +++ b/codegen/src/test/resources/software/amazon/awssdk/codegen/poet/client/c2j/presignedurl/customization.config @@ -0,0 +1,3 @@ +{ + "presignedUrlExtensionSupported": true +} \ No newline at end of file diff --git a/codegen/src/test/resources/software/amazon/awssdk/codegen/poet/client/c2j/presignedurl/service-2.json b/codegen/src/test/resources/software/amazon/awssdk/codegen/poet/client/c2j/presignedurl/service-2.json new file mode 100644 index 000000000000..cfeb496ac347 --- /dev/null +++ b/codegen/src/test/resources/software/amazon/awssdk/codegen/poet/client/c2j/presignedurl/service-2.json @@ -0,0 +1,62 @@ +{ + "version": "2.0", + "metadata": { + "apiVersion": "2010-05-08", + "endpointPrefix": "json-service-endpoint", + "globalEndpoint": "json-service.amazonaws.com", + "protocol": "json", + "serviceAbbreviation": "Json Service", + "serviceFullName": "Some Service That Uses AWS Json Protocol", + "serviceId": "Json Service", + "signingName": "json-service", + "signatureVersion": "v4", + "uid": "json-service-2010-05-08", + "xmlNamespace": "https://json-service.amazonaws.com/doc/2010-05-08/" + }, + "operations": { + "APostOperation": { + "name": "APostOperation", + "http": { + "method": "POST", + "requestUri": "/" + }, + "input": { + "shape": "APostOperationRequest" + }, + "errors": [ + { + "shape": "InvalidInputException" + } + ], + "documentation": "

Performs a post operation to the query service and has no output

" + } + }, + "shapes": { + "APostOperationRequest": { + "type": "structure", + "required": [ + "StringMember" + ], + "members": { + "StringMember": { + "shape": "String", + "documentation": "

String member

" + } + } + }, + "InvalidInputException": { + "type": "structure", + "members": { + "message": { + "shape": "String" + } + }, + "documentation": "

The request was rejected because an invalid or out-of-range value was supplied for an input parameter.

", + "exception": true + }, + "String": { + "type": "string" + } + }, + "documentation": "A service that is implemented using the json protocol" +} \ No newline at end of file diff --git a/codegen/src/test/resources/software/amazon/awssdk/codegen/poet/client/test-abstract-async-client-class-presignedurl.java b/codegen/src/test/resources/software/amazon/awssdk/codegen/poet/client/test-abstract-async-client-class-presignedurl.java new file mode 100644 index 000000000000..7e5984afccf7 --- /dev/null +++ b/codegen/src/test/resources/software/amazon/awssdk/codegen/poet/client/test-abstract-async-client-class-presignedurl.java @@ -0,0 +1,84 @@ +package software.amazon.awssdk.services.json; + +import java.util.concurrent.CompletableFuture; +import java.util.function.Function; +import software.amazon.awssdk.annotations.Generated; +import software.amazon.awssdk.annotations.SdkPublicApi; +import software.amazon.awssdk.core.SdkClient; +import software.amazon.awssdk.services.json.model.APostOperationRequest; +import software.amazon.awssdk.services.json.model.APostOperationResponse; +import software.amazon.awssdk.services.json.model.JsonRequest; +import software.amazon.awssdk.services.json.presignedurl.AsyncPresignedUrlExtension; +import software.amazon.awssdk.utils.Validate; + +@Generated("software.amazon.awssdk:codegen") +@SdkPublicApi +public abstract class DelegatingJsonAsyncClient implements JsonAsyncClient { + private final JsonAsyncClient delegate; + + public DelegatingJsonAsyncClient(JsonAsyncClient delegate) { + Validate.paramNotNull(delegate, "delegate"); + this.delegate = delegate; + } + + /** + *

+ * Performs a post operation to the query service and has no output + *

+ * + * @param aPostOperationRequest + * @return A Java Future containing the result of the APostOperation operation returned by the service.
+ * The CompletableFuture returned by this method can be completed exceptionally with the following + * exceptions. The exception returned is wrapped with CompletionException, so you need to invoke + * {@link Throwable#getCause} to retrieve the underlying exception. + * + * @sample JsonAsyncClient.APostOperation + * @see AWS + * API Documentation + */ + @Override + public CompletableFuture aPostOperation(APostOperationRequest aPostOperationRequest) { + return invokeOperation(aPostOperationRequest, request -> delegate.aPostOperation(request)); + } + + /** + * Creates an instance of {@link AsyncPresignedUrlExtension} object with the configuration set on this client. + */ + @Override + public AsyncPresignedUrlExtension presignedUrlExtension() { + return delegate.presignedUrlExtension(); + } + + @Override + public final JsonServiceClientConfiguration serviceClientConfiguration() { + return delegate.serviceClientConfiguration(); + } + + @Override + public final String serviceName() { + return delegate.serviceName(); + } + + public SdkClient delegate() { + return this.delegate; + } + + protected CompletableFuture invokeOperation(T request, + Function> operation) { + return operation.apply(request); + } + + @Override + public void close() { + delegate.close(); + } +} diff --git a/codegen/src/test/resources/software/amazon/awssdk/codegen/poet/client/test-json-async-client-interface-presignedurl.java b/codegen/src/test/resources/software/amazon/awssdk/codegen/poet/client/test-json-async-client-interface-presignedurl.java new file mode 100644 index 000000000000..4b5bfc7eb400 --- /dev/null +++ b/codegen/src/test/resources/software/amazon/awssdk/codegen/poet/client/test-json-async-client-interface-presignedurl.java @@ -0,0 +1,123 @@ +package software.amazon.awssdk.services.json; + +import java.util.concurrent.CompletableFuture; +import java.util.function.Consumer; +import software.amazon.awssdk.annotations.Generated; +import software.amazon.awssdk.annotations.SdkPublicApi; +import software.amazon.awssdk.annotations.ThreadSafe; +import software.amazon.awssdk.awscore.AwsClient; +import software.amazon.awssdk.services.json.model.APostOperationRequest; +import software.amazon.awssdk.services.json.model.APostOperationResponse; +import software.amazon.awssdk.services.json.presignedurl.AsyncPresignedUrlExtension; + +/** + * Service client for accessing Json Service asynchronously. This can be created using the static {@link #builder()} + * method.The asynchronous client performs non-blocking I/O when configured with any {@code SdkAsyncHttpClient} + * supported in the SDK. However, full non-blocking is not guaranteed as the async client may perform blocking calls in + * some cases such as credentials retrieval and endpoint discovery as part of the async API call. + * + * A service that is implemented using the json protocol + */ +@Generated("software.amazon.awssdk:codegen") +@SdkPublicApi +@ThreadSafe +public interface JsonAsyncClient extends AwsClient { + String SERVICE_NAME = "json-service"; + + /** + * Value for looking up the service's metadata from the + * {@link software.amazon.awssdk.regions.ServiceMetadataProvider}. + */ + String SERVICE_METADATA_ID = "json-service-endpoint"; + + /** + *

+ * Performs a post operation to the query service and has no output + *

+ * + * @param aPostOperationRequest + * @return A Java Future containing the result of the APostOperation operation returned by the service.
+ * The CompletableFuture returned by this method can be completed exceptionally with the following + * exceptions. The exception returned is wrapped with CompletionException, so you need to invoke + * {@link Throwable#getCause} to retrieve the underlying exception. + *
    + *
  • InvalidInputException The request was rejected because an invalid or out-of-range value was supplied + * for an input parameter.
  • + *
  • SdkException Base class for all exceptions that can be thrown by the SDK (both service and client). + * Can be used for catch all scenarios.
  • + *
  • SdkClientException If any client side error occurs such as an IO related failure, failure to get + * credentials, etc.
  • + *
  • JsonException Base class for all service exceptions. Unknown exceptions will be thrown as an instance + * of this type.
  • + *
+ * @sample JsonAsyncClient.APostOperation + * @see AWS + * API Documentation + */ + default CompletableFuture aPostOperation(APostOperationRequest aPostOperationRequest) { + throw new UnsupportedOperationException(); + } + + /** + *

+ * Performs a post operation to the query service and has no output + *

+ *
+ *

+ * This is a convenience which creates an instance of the {@link APostOperationRequest.Builder} avoiding the need to + * create one manually via {@link APostOperationRequest#builder()} + *

+ * + * @param aPostOperationRequest + * A {@link Consumer} that will call methods on + * {@link software.amazon.awssdk.services.json.model.APostOperationRequest.Builder} to create a request. + * @return A Java Future containing the result of the APostOperation operation returned by the service.
+ * The CompletableFuture returned by this method can be completed exceptionally with the following + * exceptions. The exception returned is wrapped with CompletionException, so you need to invoke + * {@link Throwable#getCause} to retrieve the underlying exception. + *
    + *
  • InvalidInputException The request was rejected because an invalid or out-of-range value was supplied + * for an input parameter.
  • + *
  • SdkException Base class for all exceptions that can be thrown by the SDK (both service and client). + * Can be used for catch all scenarios.
  • + *
  • SdkClientException If any client side error occurs such as an IO related failure, failure to get + * credentials, etc.
  • + *
  • JsonException Base class for all service exceptions. Unknown exceptions will be thrown as an instance + * of this type.
  • + *
+ * @sample JsonAsyncClient.APostOperation + * @see AWS + * API Documentation + */ + default CompletableFuture aPostOperation(Consumer aPostOperationRequest) { + return aPostOperation(APostOperationRequest.builder().applyMutation(aPostOperationRequest).build()); + } + + /** + * Creates an instance of {@link AsyncPresignedUrlExtension} object with the configuration set on this client. + */ + default AsyncPresignedUrlExtension presignedUrlExtension() { + throw new UnsupportedOperationException(); + } + + @Override + default JsonServiceClientConfiguration serviceClientConfiguration() { + throw new UnsupportedOperationException(); + } + + /** + * Create a {@link JsonAsyncClient} with the region loaded from the + * {@link software.amazon.awssdk.regions.providers.DefaultAwsRegionProviderChain} and credentials loaded from the + * {@link software.amazon.awssdk.auth.credentials.DefaultCredentialsProvider}. + */ + static JsonAsyncClient create() { + return builder().build(); + } + + /** + * Create a builder that can be used to configure and create a {@link JsonAsyncClient}. + */ + static JsonAsyncClientBuilder builder() { + return new DefaultJsonAsyncClientBuilder(); + } +} diff --git a/codegen/src/test/resources/software/amazon/awssdk/codegen/poet/client/test-presignedurl-async.java b/codegen/src/test/resources/software/amazon/awssdk/codegen/poet/client/test-presignedurl-async.java new file mode 100644 index 000000000000..5e9fdee4bcc6 --- /dev/null +++ b/codegen/src/test/resources/software/amazon/awssdk/codegen/poet/client/test-presignedurl-async.java @@ -0,0 +1,306 @@ +package software.amazon.awssdk.services.json; + +import static software.amazon.awssdk.utils.FunctionalUtils.runAndLogError; + +import java.util.Collections; +import java.util.List; +import java.util.Optional; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionException; +import java.util.function.Consumer; +import java.util.function.Function; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import software.amazon.awssdk.annotations.Generated; +import software.amazon.awssdk.annotations.SdkInternalApi; +import software.amazon.awssdk.awscore.client.config.AwsClientOption; +import software.amazon.awssdk.awscore.client.handler.AwsAsyncClientHandler; +import software.amazon.awssdk.awscore.endpoints.AwsEndpointAttribute; +import software.amazon.awssdk.awscore.endpoints.AwsEndpointProviderUtils; +import software.amazon.awssdk.awscore.endpoints.authscheme.EndpointAuthScheme; +import software.amazon.awssdk.awscore.exception.AwsServiceException; +import software.amazon.awssdk.awscore.internal.AwsProtocolMetadata; +import software.amazon.awssdk.awscore.internal.AwsServiceProtocol; +import software.amazon.awssdk.awscore.retry.AwsRetryStrategy; +import software.amazon.awssdk.core.RequestOverrideConfiguration; +import software.amazon.awssdk.core.SdkPlugin; +import software.amazon.awssdk.core.SdkRequest; +import software.amazon.awssdk.core.SelectedAuthScheme; +import software.amazon.awssdk.core.client.config.ClientOverrideConfiguration; +import software.amazon.awssdk.core.client.config.SdkClientConfiguration; +import software.amazon.awssdk.core.client.config.SdkClientOption; +import software.amazon.awssdk.core.client.handler.AsyncClientHandler; +import software.amazon.awssdk.core.client.handler.ClientExecutionParams; +import software.amazon.awssdk.core.endpoint.EndpointResolver; +import software.amazon.awssdk.core.exception.SdkClientException; +import software.amazon.awssdk.core.http.HttpResponseHandler; +import software.amazon.awssdk.core.interceptor.ExecutionAttributes; +import software.amazon.awssdk.core.interceptor.SdkInternalExecutionAttribute; +import software.amazon.awssdk.core.metrics.CoreMetric; +import software.amazon.awssdk.core.retry.RetryMode; +import software.amazon.awssdk.core.spi.identity.AuthSchemeOptionsResolver; +import software.amazon.awssdk.endpoints.Endpoint; +import software.amazon.awssdk.http.auth.spi.scheme.AuthSchemeOption; +import software.amazon.awssdk.metrics.MetricCollector; +import software.amazon.awssdk.metrics.MetricPublisher; +import software.amazon.awssdk.metrics.NoOpMetricCollector; +import software.amazon.awssdk.protocols.core.ExceptionMetadata; +import software.amazon.awssdk.protocols.json.AwsJsonProtocol; +import software.amazon.awssdk.protocols.json.AwsJsonProtocolFactory; +import software.amazon.awssdk.protocols.json.BaseAwsJsonProtocolFactory; +import software.amazon.awssdk.protocols.json.JsonOperationMetadata; +import software.amazon.awssdk.retries.api.RetryStrategy; +import software.amazon.awssdk.services.json.auth.scheme.JsonAuthSchemeParams; +import software.amazon.awssdk.services.json.auth.scheme.JsonAuthSchemeProvider; +import software.amazon.awssdk.services.json.endpoints.JsonEndpointParams; +import software.amazon.awssdk.services.json.endpoints.JsonEndpointProvider; +import software.amazon.awssdk.services.json.endpoints.internal.JsonEndpointResolverUtils; +import software.amazon.awssdk.services.json.internal.JsonServiceClientConfigurationBuilder; +import software.amazon.awssdk.services.json.internal.ServiceVersionInfo; +import software.amazon.awssdk.services.json.internal.presignedurl.DefaultAsyncPresignedUrlExtension; +import software.amazon.awssdk.services.json.model.APostOperationRequest; +import software.amazon.awssdk.services.json.model.APostOperationResponse; +import software.amazon.awssdk.services.json.model.InvalidInputException; +import software.amazon.awssdk.services.json.model.JsonException; +import software.amazon.awssdk.services.json.presignedurl.AsyncPresignedUrlExtension; +import software.amazon.awssdk.services.json.transform.APostOperationRequestMarshaller; +import software.amazon.awssdk.utils.CompletableFutureUtils; +import software.amazon.awssdk.utils.Validate; + +/** + * Internal implementation of {@link JsonAsyncClient}. + * + * @see JsonAsyncClient#builder() + */ +@Generated("software.amazon.awssdk:codegen") +@SdkInternalApi +final class DefaultJsonAsyncClient implements JsonAsyncClient { + private static final Logger log = LoggerFactory.getLogger(DefaultJsonAsyncClient.class); + + private static final AwsProtocolMetadata protocolMetadata = AwsProtocolMetadata.builder() + .serviceProtocol(AwsServiceProtocol.AWS_JSON).build(); + + private final AsyncClientHandler clientHandler; + + private final AwsJsonProtocolFactory protocolFactory; + + private final SdkClientConfiguration clientConfiguration; + + protected DefaultJsonAsyncClient(SdkClientConfiguration clientConfiguration) { + this.clientHandler = new AwsAsyncClientHandler(clientConfiguration); + this.clientConfiguration = clientConfiguration.toBuilder().option(SdkClientOption.SDK_CLIENT, this) + .option(SdkClientOption.API_METADATA, "Json_Service" + "#" + ServiceVersionInfo.VERSION).build(); + this.protocolFactory = init(AwsJsonProtocolFactory.builder()).build(); + } + + /** + *

+ * Performs a post operation to the query service and has no output + *

+ * + * @param aPostOperationRequest + * @return A Java Future containing the result of the APostOperation operation returned by the service.
+ * The CompletableFuture returned by this method can be completed exceptionally with the following + * exceptions. The exception returned is wrapped with CompletionException, so you need to invoke + * {@link Throwable#getCause} to retrieve the underlying exception. + *
    + *
  • InvalidInputException The request was rejected because an invalid or out-of-range value was supplied + * for an input parameter.
  • + *
  • SdkException Base class for all exceptions that can be thrown by the SDK (both service and client). + * Can be used for catch all scenarios.
  • + *
  • SdkClientException If any client side error occurs such as an IO related failure, failure to get + * credentials, etc.
  • + *
  • JsonException Base class for all service exceptions. Unknown exceptions will be thrown as an instance + * of this type.
  • + *
+ * @sample JsonAsyncClient.APostOperation + * @see AWS + * API Documentation + */ + @Override + public CompletableFuture aPostOperation(APostOperationRequest aPostOperationRequest) { + SdkClientConfiguration clientConfiguration = updateSdkClientConfiguration(aPostOperationRequest, this.clientConfiguration); + List metricPublishers = resolveMetricPublishers(clientConfiguration, aPostOperationRequest + .overrideConfiguration().orElse(null)); + MetricCollector apiCallMetricCollector = metricPublishers.isEmpty() ? NoOpMetricCollector.create() : MetricCollector + .create("ApiCall"); + try { + apiCallMetricCollector.reportMetric(CoreMetric.SERVICE_ID, "Json Service"); + apiCallMetricCollector.reportMetric(CoreMetric.OPERATION_NAME, "APostOperation"); + JsonOperationMetadata operationMetadata = JsonOperationMetadata.builder().hasStreamingSuccessResponse(false) + .isPayloadJson(true).build(); + + HttpResponseHandler responseHandler = protocolFactory.createResponseHandler( + operationMetadata, APostOperationResponse::builder); + Function> exceptionMetadataMapper = errorCode -> { + if (errorCode == null) { + return Optional.empty(); + } + switch (errorCode) { + case "InvalidInputException": + return Optional.of(ExceptionMetadata.builder().errorCode("InvalidInputException").httpStatusCode(400) + .exceptionBuilderSupplier(InvalidInputException::builder).build()); + default: + return Optional.empty(); + } + }; + HttpResponseHandler errorResponseHandler = createErrorResponseHandler(protocolFactory, + operationMetadata, exceptionMetadataMapper); + + CompletableFuture executeFuture = clientHandler + .execute(new ClientExecutionParams() + .withOperationName("APostOperation").withProtocolMetadata(protocolMetadata) + .withMarshaller(new APostOperationRequestMarshaller(protocolFactory)) + .withResponseHandler(responseHandler).withErrorResponseHandler(errorResponseHandler) + .withRequestConfiguration(clientConfiguration).withMetricCollector(apiCallMetricCollector) + .withAuthSchemeOptionsResolver(authSchemeResolver("APostOperation", clientConfiguration)) + .withEndpointResolver(endpointResolver("APostOperation")).withInput(aPostOperationRequest)); + CompletableFuture whenCompleted = executeFuture.whenComplete((r, e) -> { + metricPublishers.forEach(p -> p.publish(apiCallMetricCollector.collect())); + }); + executeFuture = CompletableFutureUtils.forwardExceptionTo(whenCompleted, executeFuture); + return executeFuture; + } catch (Throwable t) { + metricPublishers.forEach(p -> p.publish(apiCallMetricCollector.collect())); + return CompletableFutureUtils.failedFuture(t); + } + } + + @Override + public AsyncPresignedUrlExtension presignedUrlExtension() { + return new DefaultAsyncPresignedUrlExtension(clientHandler, protocolFactory, clientConfiguration, protocolMetadata); + } + + @Override + public final JsonServiceClientConfiguration serviceClientConfiguration() { + return new JsonServiceClientConfigurationBuilder(this.clientConfiguration.toBuilder()).build(); + } + + @Override + public final String serviceName() { + return SERVICE_NAME; + } + + private > T init(T builder) { + return builder.clientConfiguration(clientConfiguration).defaultServiceExceptionSupplier(JsonException::builder) + .protocol(AwsJsonProtocol.AWS_JSON).protocolVersion("1.1"); + } + + private static List resolveMetricPublishers(SdkClientConfiguration clientConfiguration, + RequestOverrideConfiguration requestOverrideConfiguration) { + List publishers = null; + if (requestOverrideConfiguration != null) { + publishers = requestOverrideConfiguration.metricPublishers(); + } + if (publishers == null || publishers.isEmpty()) { + publishers = clientConfiguration.option(SdkClientOption.METRIC_PUBLISHERS); + } + if (publishers == null) { + publishers = Collections.emptyList(); + } + return publishers; + } + + private List resolveAuthSchemeOptions(SdkRequest request, String operationName, + SdkClientConfiguration clientConfiguration) { + JsonAuthSchemeProvider requestAuthSchemeProvider = request + .overrideConfiguration() + .flatMap(c -> c.authSchemeProvider()) + .map(p -> Validate + .isInstanceOf(JsonAuthSchemeProvider.class, p, "Expected an instance of JsonAuthSchemeProvider")) + .orElse(null); + JsonAuthSchemeProvider authSchemeProvider = requestAuthSchemeProvider != null ? requestAuthSchemeProvider : Validate + .isInstanceOf(JsonAuthSchemeProvider.class, clientConfiguration.option(SdkClientOption.AUTH_SCHEME_PROVIDER), + "Expected an instance of JsonAuthSchemeProvider"); + JsonAuthSchemeParams.Builder paramsBuilder = JsonAuthSchemeParams.builder().operation(operationName); + paramsBuilder.region(clientConfiguration.option(AwsClientOption.AWS_REGION)); + List options = authSchemeProvider.resolveAuthScheme(paramsBuilder.build()); + return options; + } + + private Endpoint resolveEndpoint(SdkRequest request, ExecutionAttributes executionAttributes, String operationName) { + JsonEndpointProvider provider = (JsonEndpointProvider) executionAttributes + .getAttribute(SdkInternalExecutionAttribute.ENDPOINT_PROVIDER); + try { + JsonEndpointParams endpointParams = JsonEndpointResolverUtils.ruleParams(request, executionAttributes); + Endpoint endpoint = provider.resolveEndpoint(endpointParams).join(); + if (!AwsEndpointProviderUtils.disableHostPrefixInjection(executionAttributes)) { + Optional hostPrefix = JsonEndpointResolverUtils.hostPrefix(operationName, request); + if (hostPrefix.isPresent()) { + endpoint = AwsEndpointProviderUtils.addHostPrefix(endpoint, hostPrefix.get()); + } + } + List endpointAuthSchemes = endpoint.attribute(AwsEndpointAttribute.AUTH_SCHEMES); + SelectedAuthScheme selectedAuthScheme = executionAttributes + .getAttribute(SdkInternalExecutionAttribute.SELECTED_AUTH_SCHEME); + if (endpointAuthSchemes != null && selectedAuthScheme != null) { + selectedAuthScheme = JsonEndpointResolverUtils.authSchemeWithEndpointSignerProperties(endpointAuthSchemes, + selectedAuthScheme); + executionAttributes.putAttribute(SdkInternalExecutionAttribute.SELECTED_AUTH_SCHEME, selectedAuthScheme); + } + JsonEndpointResolverUtils.setMetricValues(endpoint, executionAttributes); + return endpoint; + } catch (CompletionException e) { + Throwable cause = e.getCause(); + if (cause instanceof SdkClientException) { + throw (SdkClientException) cause; + } + throw SdkClientException.create("Endpoint resolution failed: " + cause.getMessage(), cause); + } + } + + private AuthSchemeOptionsResolver authSchemeResolver(String operationName, SdkClientConfiguration clientConfiguration) { + return r -> resolveAuthSchemeOptions(r, operationName, clientConfiguration); + } + + private EndpointResolver endpointResolver(String operationName) { + return (r, a) -> resolveEndpoint(r, a, operationName); + } + + private void updateRetryStrategyClientConfiguration(SdkClientConfiguration.Builder configuration) { + ClientOverrideConfiguration.Builder builder = configuration.asOverrideConfigurationBuilder(); + RetryMode retryMode = builder.retryMode(); + if (retryMode != null) { + configuration.option(SdkClientOption.RETRY_STRATEGY, AwsRetryStrategy.forRetryMode(retryMode)); + } else { + Consumer> configurator = builder.retryStrategyConfigurator(); + if (configurator != null) { + RetryStrategy.Builder defaultBuilder = AwsRetryStrategy.defaultRetryStrategy().toBuilder(); + configurator.accept(defaultBuilder); + configuration.option(SdkClientOption.RETRY_STRATEGY, defaultBuilder.build()); + } else { + RetryStrategy retryStrategy = builder.retryStrategy(); + if (retryStrategy != null) { + configuration.option(SdkClientOption.RETRY_STRATEGY, retryStrategy); + } + } + } + configuration.option(SdkClientOption.CONFIGURED_RETRY_MODE, null); + configuration.option(SdkClientOption.CONFIGURED_RETRY_STRATEGY, null); + configuration.option(SdkClientOption.CONFIGURED_RETRY_CONFIGURATOR, null); + } + + private SdkClientConfiguration updateSdkClientConfiguration(SdkRequest request, SdkClientConfiguration clientConfiguration) { + List plugins = request.overrideConfiguration().map(c -> c.plugins()).orElse(Collections.emptyList()); + if (plugins.isEmpty()) { + return clientConfiguration; + } + SdkClientConfiguration.Builder configuration = clientConfiguration.toBuilder(); + JsonServiceClientConfigurationBuilder serviceConfigBuilder = new JsonServiceClientConfigurationBuilder(configuration); + for (SdkPlugin plugin : plugins) { + plugin.configureClient(serviceConfigBuilder); + } + updateRetryStrategyClientConfiguration(configuration); + return configuration.build(); + } + + private HttpResponseHandler createErrorResponseHandler(BaseAwsJsonProtocolFactory protocolFactory, + JsonOperationMetadata operationMetadata, Function> exceptionMetadataMapper) { + return protocolFactory.createErrorResponseHandler(operationMetadata, exceptionMetadataMapper); + } + + @Override + public void close() { + clientHandler.close(); + } +} diff --git a/codegen/src/test/resources/software/amazon/awssdk/codegen/poet/rules/endpoint-resolve-interceptor-preSra.java b/codegen/src/test/resources/software/amazon/awssdk/codegen/poet/rules/endpoint-resolve-interceptor-preSra.java index cf27b03ed0c3..de8be3aa52a1 100644 --- a/codegen/src/test/resources/software/amazon/awssdk/codegen/poet/rules/endpoint-resolve-interceptor-preSra.java +++ b/codegen/src/test/resources/software/amazon/awssdk/codegen/poet/rules/endpoint-resolve-interceptor-preSra.java @@ -62,7 +62,7 @@ public QueryResolveEndpointInterceptor() { @Override public SdkRequest modifyRequest(Context.ModifyRequest context, ExecutionAttributes executionAttributes) { SdkRequest result = context.request(); - if (AwsEndpointProviderUtils.endpointIsDiscovered(executionAttributes)) { + if (AwsEndpointProviderUtils.skipEndpointResolution(executionAttributes)) { return result; } QueryEndpointProvider provider = (QueryEndpointProvider) executionAttributes @@ -110,7 +110,7 @@ public SdkRequest modifyRequest(Context.ModifyRequest context, ExecutionAttribut @Override public SdkHttpRequest modifyHttpRequest(Context.ModifyHttpRequest context, ExecutionAttributes executionAttributes) { Endpoint resolvedEndpoint = executionAttributes.getAttribute(SdkInternalExecutionAttribute.RESOLVED_ENDPOINT); - if (resolvedEndpoint.headers().isEmpty()) { + if (resolvedEndpoint == null || CollectionUtils.isNullOrEmpty(resolvedEndpoint.headers())) { return context.httpRequest(); } SdkHttpRequest.Builder httpRequestBuilder = context.httpRequest().toBuilder(); diff --git a/codegen/src/test/resources/software/amazon/awssdk/codegen/poet/rules/endpoint-resolve-interceptor-with-endpointsbasedauth.java b/codegen/src/test/resources/software/amazon/awssdk/codegen/poet/rules/endpoint-resolve-interceptor-with-endpointsbasedauth.java index 6d6aa1dc2138..71cd3af7868b 100644 --- a/codegen/src/test/resources/software/amazon/awssdk/codegen/poet/rules/endpoint-resolve-interceptor-with-endpointsbasedauth.java +++ b/codegen/src/test/resources/software/amazon/awssdk/codegen/poet/rules/endpoint-resolve-interceptor-with-endpointsbasedauth.java @@ -50,7 +50,7 @@ public final class QueryResolveEndpointInterceptor implements ExecutionIntercept @Override public SdkRequest modifyRequest(Context.ModifyRequest context, ExecutionAttributes executionAttributes) { SdkRequest result = context.request(); - if (AwsEndpointProviderUtils.endpointIsDiscovered(executionAttributes)) { + if (AwsEndpointProviderUtils.skipEndpointResolution(executionAttributes)) { return result; } QueryEndpointProvider provider = (QueryEndpointProvider) executionAttributes @@ -102,7 +102,7 @@ public SdkRequest modifyRequest(Context.ModifyRequest context, ExecutionAttribut @Override public SdkHttpRequest modifyHttpRequest(Context.ModifyHttpRequest context, ExecutionAttributes executionAttributes) { Endpoint resolvedEndpoint = executionAttributes.getAttribute(SdkInternalExecutionAttribute.RESOLVED_ENDPOINT); - if (resolvedEndpoint.headers().isEmpty()) { + if (resolvedEndpoint == null || CollectionUtils.isNullOrEmpty(resolvedEndpoint.headers())) { return context.httpRequest(); } SdkHttpRequest.Builder httpRequestBuilder = context.httpRequest().toBuilder(); diff --git a/codegen/src/test/resources/software/amazon/awssdk/codegen/poet/rules/endpoint-resolve-interceptor-with-multiauthsigv4a.java b/codegen/src/test/resources/software/amazon/awssdk/codegen/poet/rules/endpoint-resolve-interceptor-with-multiauthsigv4a.java index 858d39fc7a69..26e225ab6fc5 100644 --- a/codegen/src/test/resources/software/amazon/awssdk/codegen/poet/rules/endpoint-resolve-interceptor-with-multiauthsigv4a.java +++ b/codegen/src/test/resources/software/amazon/awssdk/codegen/poet/rules/endpoint-resolve-interceptor-with-multiauthsigv4a.java @@ -39,7 +39,7 @@ public final class DatabaseResolveEndpointInterceptor implements ExecutionInterc @Override public SdkRequest modifyRequest(Context.ModifyRequest context, ExecutionAttributes executionAttributes) { SdkRequest result = context.request(); - if (AwsEndpointProviderUtils.endpointIsDiscovered(executionAttributes)) { + if (AwsEndpointProviderUtils.skipEndpointResolution(executionAttributes)) { return result; } DatabaseEndpointProvider provider = (DatabaseEndpointProvider) executionAttributes @@ -91,7 +91,7 @@ public SdkRequest modifyRequest(Context.ModifyRequest context, ExecutionAttribut @Override public SdkHttpRequest modifyHttpRequest(Context.ModifyHttpRequest context, ExecutionAttributes executionAttributes) { Endpoint resolvedEndpoint = executionAttributes.getAttribute(SdkInternalExecutionAttribute.RESOLVED_ENDPOINT); - if (resolvedEndpoint.headers().isEmpty()) { + if (resolvedEndpoint == null || CollectionUtils.isNullOrEmpty(resolvedEndpoint.headers())) { return context.httpRequest(); } SdkHttpRequest.Builder httpRequestBuilder = context.httpRequest().toBuilder(); diff --git a/codegen/src/test/resources/software/amazon/awssdk/codegen/poet/rules/endpoint-resolve-interceptor-with-stringarray.java b/codegen/src/test/resources/software/amazon/awssdk/codegen/poet/rules/endpoint-resolve-interceptor-with-stringarray.java index 0f5033376ffc..23ba5b360972 100644 --- a/codegen/src/test/resources/software/amazon/awssdk/codegen/poet/rules/endpoint-resolve-interceptor-with-stringarray.java +++ b/codegen/src/test/resources/software/amazon/awssdk/codegen/poet/rules/endpoint-resolve-interceptor-with-stringarray.java @@ -41,7 +41,7 @@ public final class SampleSvcResolveEndpointInterceptor implements ExecutionInter @Override public SdkRequest modifyRequest(Context.ModifyRequest context, ExecutionAttributes executionAttributes) { SdkRequest result = context.request(); - if (AwsEndpointProviderUtils.endpointIsDiscovered(executionAttributes)) { + if (AwsEndpointProviderUtils.skipEndpointResolution(executionAttributes)) { return result; } SampleSvcEndpointProvider provider = (SampleSvcEndpointProvider) executionAttributes @@ -84,7 +84,7 @@ public SdkRequest modifyRequest(Context.ModifyRequest context, ExecutionAttribut @Override public SdkHttpRequest modifyHttpRequest(Context.ModifyHttpRequest context, ExecutionAttributes executionAttributes) { Endpoint resolvedEndpoint = executionAttributes.getAttribute(SdkInternalExecutionAttribute.RESOLVED_ENDPOINT); - if (resolvedEndpoint.headers().isEmpty()) { + if (resolvedEndpoint == null || CollectionUtils.isNullOrEmpty(resolvedEndpoint.headers())) { return context.httpRequest(); } SdkHttpRequest.Builder httpRequestBuilder = context.httpRequest().toBuilder(); diff --git a/codegen/src/test/resources/software/amazon/awssdk/codegen/poet/rules/endpoint-resolve-interceptor.java b/codegen/src/test/resources/software/amazon/awssdk/codegen/poet/rules/endpoint-resolve-interceptor.java index abdfedab0455..1d170e6413e8 100644 --- a/codegen/src/test/resources/software/amazon/awssdk/codegen/poet/rules/endpoint-resolve-interceptor.java +++ b/codegen/src/test/resources/software/amazon/awssdk/codegen/poet/rules/endpoint-resolve-interceptor.java @@ -50,7 +50,7 @@ public final class QueryResolveEndpointInterceptor implements ExecutionIntercept @Override public SdkRequest modifyRequest(Context.ModifyRequest context, ExecutionAttributes executionAttributes) { SdkRequest result = context.request(); - if (AwsEndpointProviderUtils.endpointIsDiscovered(executionAttributes)) { + if (AwsEndpointProviderUtils.skipEndpointResolution(executionAttributes)) { return result; } QueryEndpointProvider provider = (QueryEndpointProvider) executionAttributes @@ -93,7 +93,7 @@ public SdkRequest modifyRequest(Context.ModifyRequest context, ExecutionAttribut @Override public SdkHttpRequest modifyHttpRequest(Context.ModifyHttpRequest context, ExecutionAttributes executionAttributes) { Endpoint resolvedEndpoint = executionAttributes.getAttribute(SdkInternalExecutionAttribute.RESOLVED_ENDPOINT); - if (resolvedEndpoint.headers().isEmpty()) { + if (resolvedEndpoint == null || CollectionUtils.isNullOrEmpty(resolvedEndpoint.headers())) { return context.httpRequest(); } SdkHttpRequest.Builder httpRequestBuilder = context.httpRequest().toBuilder(); diff --git a/codegen/src/test/resources/software/amazon/awssdk/codegen/poet/rules/request-set-endpoint-interceptor.java b/codegen/src/test/resources/software/amazon/awssdk/codegen/poet/rules/request-set-endpoint-interceptor.java index 4cfeedb59e46..8895321a6863 100644 --- a/codegen/src/test/resources/software/amazon/awssdk/codegen/poet/rules/request-set-endpoint-interceptor.java +++ b/codegen/src/test/resources/software/amazon/awssdk/codegen/poet/rules/request-set-endpoint-interceptor.java @@ -14,7 +14,7 @@ public final class QueryRequestSetEndpointInterceptor implements ExecutionInterceptor { @Override public SdkHttpRequest modifyHttpRequest(Context.ModifyHttpRequest context, ExecutionAttributes executionAttributes) { - if (AwsEndpointProviderUtils.endpointIsDiscovered(executionAttributes)) { + if (AwsEndpointProviderUtils.skipEndpointResolution(executionAttributes)) { return context.httpRequest(); } Endpoint endpoint = executionAttributes.getAttribute(SdkInternalExecutionAttribute.RESOLVED_ENDPOINT); diff --git a/core/aws-core/src/main/java/software/amazon/awssdk/awscore/endpoints/AwsEndpointProviderUtils.java b/core/aws-core/src/main/java/software/amazon/awssdk/awscore/endpoints/AwsEndpointProviderUtils.java index 53555c1adb53..4fd4c1c972d2 100644 --- a/core/aws-core/src/main/java/software/amazon/awssdk/awscore/endpoints/AwsEndpointProviderUtils.java +++ b/core/aws-core/src/main/java/software/amazon/awssdk/awscore/endpoints/AwsEndpointProviderUtils.java @@ -80,11 +80,24 @@ public static boolean endpointIsOverridden(ExecutionAttributes attrs) { /** * True if the {@link SdkInternalExecutionAttribute#IS_DISCOVERED_ENDPOINT} attribute is present and its value * is {@code true}, {@code false} otherwise. + * + * @deprecated Use {@link #skipEndpointResolution(ExecutionAttributes)} instead. */ + @Deprecated public static boolean endpointIsDiscovered(ExecutionAttributes attrs) { return attrs.getOptionalAttribute(SdkInternalExecutionAttribute.IS_DISCOVERED_ENDPOINT).orElse(false); } + /** + * True if endpoint resolution should be skipped for the current request. This returns {@code true} if either + * {@link SdkInternalExecutionAttribute#SKIP_ENDPOINT_RESOLUTION} or + * {@link SdkInternalExecutionAttribute#IS_DISCOVERED_ENDPOINT} is set to {@code true}. + */ + public static boolean skipEndpointResolution(ExecutionAttributes attrs) { + return attrs.getOptionalAttribute(SdkInternalExecutionAttribute.SKIP_ENDPOINT_RESOLUTION).orElse(false) + || attrs.getOptionalAttribute(SdkInternalExecutionAttribute.IS_DISCOVERED_ENDPOINT).orElse(false); + } + /** * True if the {@link SdkInternalExecutionAttribute#DISABLE_HOST_PREFIX_INJECTION} attribute is present and its * value is {@code true}, {@code false} otherwise. diff --git a/core/aws-core/src/test/java/software/amazon/awssdk/awscore/endpoints/AwsEndpointProviderUtilsTest.java b/core/aws-core/src/test/java/software/amazon/awssdk/awscore/endpoints/AwsEndpointProviderUtilsTest.java index b1fa2b82d6a1..821e584f37a6 100644 --- a/core/aws-core/src/test/java/software/amazon/awssdk/awscore/endpoints/AwsEndpointProviderUtilsTest.java +++ b/core/aws-core/src/test/java/software/amazon/awssdk/awscore/endpoints/AwsEndpointProviderUtilsTest.java @@ -32,6 +32,26 @@ class AwsEndpointProviderUtilsTest { + @Test + void skipEndpointResolution_skipAttrIsTrue_returnsTrue() { + ExecutionAttributes attrs = new ExecutionAttributes(); + attrs.putAttribute(SdkInternalExecutionAttribute.SKIP_ENDPOINT_RESOLUTION, true); + assertThat(AwsEndpointProviderUtils.skipEndpointResolution(attrs)).isTrue(); + } + + @Test + void skipEndpointResolution_discoveredEndpointIsTrue_returnsTrue() { + ExecutionAttributes attrs = new ExecutionAttributes(); + attrs.putAttribute(SdkInternalExecutionAttribute.IS_DISCOVERED_ENDPOINT, true); + assertThat(AwsEndpointProviderUtils.skipEndpointResolution(attrs)).isTrue(); + } + + @Test + void skipEndpointResolution_attrsAbsent_returnsFalse() { + ExecutionAttributes attrs = new ExecutionAttributes(); + assertThat(AwsEndpointProviderUtils.skipEndpointResolution(attrs)).isFalse(); + } + @Test void regionBuiltIn_returnsAttrValue() { ExecutionAttributes attrs = new ExecutionAttributes(); diff --git a/core/checksums/src/main/java/software/amazon/awssdk/checksums/DefaultChecksumAlgorithm.java b/core/checksums/src/main/java/software/amazon/awssdk/checksums/DefaultChecksumAlgorithm.java index 65a2940b8aa2..dae3f27db564 100644 --- a/core/checksums/src/main/java/software/amazon/awssdk/checksums/DefaultChecksumAlgorithm.java +++ b/core/checksums/src/main/java/software/amazon/awssdk/checksums/DefaultChecksumAlgorithm.java @@ -15,6 +15,8 @@ package software.amazon.awssdk.checksums; +import java.util.Collection; +import java.util.Collections; import java.util.Locale; import java.util.concurrent.ConcurrentHashMap; import software.amazon.awssdk.annotations.SdkProtectedApi; @@ -52,6 +54,13 @@ public static ChecksumAlgorithm fromValue(String algorithm) { return ChecksumAlgorithmsCache.VALUES.get(algorithm.toUpperCase(Locale.US)); } + /** + * Returns all registered checksum algorithms. + */ + public static Collection values() { + return Collections.unmodifiableCollection(ChecksumAlgorithmsCache.VALUES.values()); + } + private static final class ChecksumAlgorithmsCache { private static final ConcurrentHashMap VALUES = new ConcurrentHashMap<>(); diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/SplittingTransformerConfiguration.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/SplittingTransformerConfiguration.java index 766213195203..21b564983c33 100644 --- a/core/sdk-core/src/main/java/software/amazon/awssdk/core/SplittingTransformerConfiguration.java +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/SplittingTransformerConfiguration.java @@ -16,6 +16,7 @@ package software.amazon.awssdk.core; import java.util.Objects; +import java.util.function.UnaryOperator; import software.amazon.awssdk.annotations.SdkPublicApi; import software.amazon.awssdk.core.async.AsyncResponseTransformer; import software.amazon.awssdk.core.internal.async.SplittingTransformer; @@ -35,9 +36,11 @@ public final class SplittingTransformerConfiguration implements ToCopyableBuilde SplittingTransformerConfiguration> { private final Long bufferSizeInBytes; + private final UnaryOperator responseMapper; private SplittingTransformerConfiguration(DefaultBuilder builder) { this.bufferSizeInBytes = Validate.paramNotNull(builder.bufferSize, "bufferSize"); + this.responseMapper = builder.responseMapper; } /** @@ -54,6 +57,14 @@ public Long bufferSizeInBytes() { return bufferSizeInBytes; } + /** + * @return the response mapper applied to the first response before delivery to the upstream transformer, or null if + * not set. See {@link Builder#responseMapper(UnaryOperator)} for semantics. + */ + public UnaryOperator responseMapper() { + return responseMapper; + } + @Override public boolean equals(Object o) { if (this == o) { @@ -65,12 +76,15 @@ public boolean equals(Object o) { SplittingTransformerConfiguration that = (SplittingTransformerConfiguration) o; - return Objects.equals(bufferSizeInBytes, that.bufferSizeInBytes); + return Objects.equals(bufferSizeInBytes, that.bufferSizeInBytes) + && Objects.equals(responseMapper, that.responseMapper); } @Override public int hashCode() { - return bufferSizeInBytes != null ? bufferSizeInBytes.hashCode() : 0; + int result = bufferSizeInBytes != null ? bufferSizeInBytes.hashCode() : 0; + result = 31 * result + (responseMapper != null ? responseMapper.hashCode() : 0); + return result; } @Override @@ -94,13 +108,29 @@ public interface Builder extends CopyableBuilderOnly applied by the default {@code split} implementation. A transformer that overrides {@code split} (such as a + * parallel, file-based one) may not read it, in which case it has no effect. + * + * @param responseMapper a function to transform the response before delivery, or null for no mapping + * @return This object for method chaining. + */ + Builder responseMapper(UnaryOperator responseMapper); } private static final class DefaultBuilder implements Builder { private Long bufferSize; + private UnaryOperator responseMapper; private DefaultBuilder(SplittingTransformerConfiguration configuration) { this.bufferSize = configuration.bufferSizeInBytes; + this.responseMapper = configuration.responseMapper; } private DefaultBuilder() { @@ -112,6 +142,12 @@ public Builder bufferSizeInBytes(Long bufferSize) { return this; } + @Override + public Builder responseMapper(UnaryOperator responseMapper) { + this.responseMapper = responseMapper; + return this; + } + @Override public SplittingTransformerConfiguration build() { return new SplittingTransformerConfiguration(this); diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/async/AsyncResponseTransformer.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/async/AsyncResponseTransformer.java index 4c27501e6340..dd4a5140170f 100644 --- a/core/sdk-core/src/main/java/software/amazon/awssdk/core/async/AsyncResponseTransformer.java +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/async/AsyncResponseTransformer.java @@ -138,6 +138,7 @@ default SplitResult split(SplittingTransformerConfiguration .builder() .upstreamResponseTransformer(this) .maximumBufferSizeInBytes(splitConfig.bufferSizeInBytes()) + .responseMapper(splitConfig.responseMapper()) .resultFuture(future) .build(); return AsyncResponseTransformer.SplitResult.builder() diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/interceptor/SdkInternalExecutionAttribute.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/interceptor/SdkInternalExecutionAttribute.java index cbcaec2c9333..eea10003c89b 100644 --- a/core/sdk-core/src/main/java/software/amazon/awssdk/core/interceptor/SdkInternalExecutionAttribute.java +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/interceptor/SdkInternalExecutionAttribute.java @@ -130,10 +130,22 @@ public final class SdkInternalExecutionAttribute extends SdkExecutionAttribute { /** * Whether the endpoint on the request is the result of Endpoint Discovery. + * + * @deprecated This attribute is specific to the endpoint discovery feature and should not be used to skip endpoint + * resolution; use {@link #SKIP_ENDPOINT_RESOLUTION} instead. */ + @Deprecated public static final ExecutionAttribute IS_DISCOVERED_ENDPOINT = new ExecutionAttribute<>("IsDiscoveredEndpoint"); + /** + * Whether endpoint resolution should be skipped for the current request. When set to {@code true}, the SDK will not + * invoke the endpoint provider and will use the endpoint already set on the request. This is used for pre-signed URL + * operations and other scenarios where the endpoint is already fully resolved. + */ + public static final ExecutionAttribute SKIP_ENDPOINT_RESOLUTION = + new ExecutionAttribute<>("SkipEndpointResolution"); + /** * The nano time that the current API call attempt began. */ diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/async/ByteArrayAsyncResponseTransformer.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/async/ByteArrayAsyncResponseTransformer.java index e250eab650b0..3c94080f604c 100644 --- a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/async/ByteArrayAsyncResponseTransformer.java +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/async/ByteArrayAsyncResponseTransformer.java @@ -72,7 +72,7 @@ public void exceptionOccurred(Throwable throwable) { public SplitResult> split(SplittingTransformerConfiguration splitConfig) { CompletableFuture> future = new CompletableFuture<>(); SdkPublisher> transformer = - new ByteArraySplittingTransformer<>(this, future); + new ByteArraySplittingTransformer<>(this, future, splitConfig.responseMapper()); return AsyncResponseTransformer.SplitResult.>builder() .publisher(transformer) .resultFuture(future) diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/async/ByteArraySplittingTransformer.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/async/ByteArraySplittingTransformer.java index 2531f7f8166b..fa5761d01bd2 100644 --- a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/async/ByteArraySplittingTransformer.java +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/async/ByteArraySplittingTransformer.java @@ -23,10 +23,12 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReference; +import java.util.function.UnaryOperator; import org.reactivestreams.Subscriber; import org.reactivestreams.Subscription; import software.amazon.awssdk.annotations.SdkInternalApi; import software.amazon.awssdk.core.ResponseBytes; +import software.amazon.awssdk.core.SdkResponse; import software.amazon.awssdk.core.async.AsyncResponseTransformer; import software.amazon.awssdk.core.async.SdkPublisher; import software.amazon.awssdk.core.exception.SdkClientException; @@ -84,12 +86,30 @@ public class ByteArraySplittingTransformer implements SdkPublisher buffers; + private final UnaryOperator responseMapper; + public ByteArraySplittingTransformer(AsyncResponseTransformer> upstreamResponseTransformer, CompletableFuture> resultFuture) { + this(upstreamResponseTransformer, resultFuture, UnaryOperator.identity()); + } + + public ByteArraySplittingTransformer(AsyncResponseTransformer> + upstreamResponseTransformer, + CompletableFuture> resultFuture, + UnaryOperator responseMapper) { this.upstreamResponseTransformer = upstreamResponseTransformer; this.resultFuture = resultFuture; this.buffers = new ConcurrentHashMap<>(); + this.responseMapper = responseMapper != null ? responseMapper : UnaryOperator.identity(); + } + + @SuppressWarnings("unchecked") + private ResponseT mapResponse(ResponseT response) { + if (!(response instanceof SdkResponse)) { + return response; + } + return (ResponseT) responseMapper.apply((SdkResponse) response); } @Override @@ -181,7 +201,7 @@ private void handleSubscriptionCancel() { CompletableFuture> upstreamPrepareFuture = upstreamResponseTransformer.prepare(); CompletableFutureUtils.forwardResultTo(upstreamPrepareFuture, resultFuture); - upstreamResponseTransformer.onResponse(responseT.get()); + upstreamResponseTransformer.onResponse(mapResponse(responseT.get())); int totalPartCount = nextPartNumber.get() - 1; if (buffers.size() != totalPartCount) { diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/async/SplittingTransformer.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/async/SplittingTransformer.java index d4cf1c7a2356..0b94dd2f7c34 100644 --- a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/async/SplittingTransformer.java +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/async/SplittingTransformer.java @@ -19,9 +19,11 @@ import java.util.concurrent.CompletableFuture; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; +import java.util.function.UnaryOperator; import org.reactivestreams.Subscriber; import org.reactivestreams.Subscription; import software.amazon.awssdk.annotations.SdkInternalApi; +import software.amazon.awssdk.core.SdkResponse; import software.amazon.awssdk.core.SplittingTransformerConfiguration; import software.amazon.awssdk.core.async.AsyncResponseTransformer; import software.amazon.awssdk.core.async.SdkPublisher; @@ -112,16 +114,18 @@ public class SplittingTransformer implements SdkPublisher upstreamResponseTransformer, - Long maximumBufferSizeInBytes, - CompletableFuture resultFuture) { + private final UnaryOperator responseMapper; + + private SplittingTransformer(Builder builder) { this.upstreamResponseTransformer = Validate.paramNotNull( - upstreamResponseTransformer, "upstreamResponseTransformer"); - this.resultFuture = Validate.paramNotNull( - resultFuture, "resultFuture"); - Validate.notNull(maximumBufferSizeInBytes, "maximumBufferSizeInBytes"); + builder.upstreamResponseTransformer, "upstreamResponseTransformer"); + this.resultFuture = Validate.paramNotNull(builder.returnFuture, "resultFuture"); + Validate.notNull(builder.maximumBufferSize, "maximumBufferSizeInBytes"); this.maximumBufferInBytes = Validate.isPositive( - maximumBufferSizeInBytes, "maximumBufferSizeInBytes"); + builder.maximumBufferSize, "maximumBufferSizeInBytes"); + this.responseMapper = builder.responseMapper != null + ? builder.responseMapper + : UnaryOperator.identity(); this.resultFuture.whenComplete((r, e) -> { if (e == null) { @@ -133,6 +137,14 @@ private SplittingTransformer(AsyncResponseTransformer upstre }); } + @SuppressWarnings("unchecked") + private ResponseT mapResponse(ResponseT response) { + if (!(response instanceof SdkResponse)) { + return response; + } + return (ResponseT) responseMapper.apply((SdkResponse) response); + } + /** * @param downstreamSubscriber the {@link Subscriber} to the individual AsyncResponseTransformer */ @@ -296,7 +308,7 @@ public CompletableFuture prepare() { public void onResponse(ResponseT response) { if (onResponseCalled.compareAndSet(false, true)) { log.trace(() -> "calling onResponse on the upstream transformer"); - upstreamResponseTransformer.onResponse(response); + upstreamResponseTransformer.onResponse(mapResponse(response)); } this.response = response; } @@ -393,6 +405,7 @@ public static final class Builder { private Long maximumBufferSize; private CompletableFuture returnFuture; private AsyncResponseTransformer upstreamResponseTransformer; + private UnaryOperator responseMapper; private Builder() { } @@ -437,10 +450,13 @@ public Builder resultFuture(CompletableFuture retur return this; } + public Builder responseMapper(UnaryOperator responseMapper) { + this.responseMapper = responseMapper; + return this; + } + public SplittingTransformer build() { - return new SplittingTransformer<>(this.upstreamResponseTransformer, - this.maximumBufferSize, - this.returnFuture); + return new SplittingTransformer<>(this); } } } diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/handler/BaseClientHandler.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/handler/BaseClientHandler.java index 41cb4f6477dc..3c7f2d58a2a8 100644 --- a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/handler/BaseClientHandler.java +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/handler/BaseClientHandler.java @@ -103,6 +103,8 @@ private static SdkHttpFullRequest modifyEndpointHostIfNeeded(SdkHttpFullRequest ClientExecutionParams executionParams) { if (executionParams.discoveredEndpoint() != null) { URI discoveredEndpoint = executionParams.discoveredEndpoint(); + executionParams.putExecutionAttribute(SdkInternalExecutionAttribute.SKIP_ENDPOINT_RESOLUTION, true); + // Keep deprecated attribute for cross-module compatibility with older generated service clients executionParams.putExecutionAttribute(SdkInternalExecutionAttribute.IS_DISCOVERED_ENDPOINT, true); return originalRequest.toBuilder().host(discoveredEndpoint.getHost()).port(discoveredEndpoint.getPort()).build(); } diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/pipeline/stages/EndpointResolutionStage.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/pipeline/stages/EndpointResolutionStage.java index 183ace09971c..0bf7ff20a0aa 100644 --- a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/pipeline/stages/EndpointResolutionStage.java +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/pipeline/stages/EndpointResolutionStage.java @@ -61,6 +61,10 @@ public SdkHttpFullRequest.Builder execute(SdkHttpFullRequest.Builder request, Re return request; } + if (Boolean.TRUE.equals(attrs.getAttribute(SdkInternalExecutionAttribute.SKIP_ENDPOINT_RESOLUTION))) { + return request; + } + // Skip if endpoint was already resolved by an old service interceptor Endpoint existingEndpoint = attrs.getAttribute(SdkInternalExecutionAttribute.RESOLVED_ENDPOINT); if (existingEndpoint != null) { diff --git a/services-custom/s3-transfer-manager/src/it/java/software/amazon/awssdk/transfer/s3/S3IntegrationTestBase.java b/services-custom/s3-transfer-manager/src/it/java/software/amazon/awssdk/transfer/s3/S3IntegrationTestBase.java index 07ca3eb0cb45..9385b577f105 100644 --- a/services-custom/s3-transfer-manager/src/it/java/software/amazon/awssdk/transfer/s3/S3IntegrationTestBase.java +++ b/services-custom/s3-transfer-manager/src/it/java/software/amazon/awssdk/transfer/s3/S3IntegrationTestBase.java @@ -57,11 +57,13 @@ public class S3IntegrationTestBase extends AwsTestBase { protected static S3Client s3; protected static S3AsyncClient s3Async; + protected static S3AsyncClient s3NonMultipartAsync; protected static S3AsyncClient s3CrtAsync; protected static S3TransferManager tmCrt; protected static S3TransferManager tmJava; + protected static S3TransferManager tmNonMultipartJava; /** * Loads the AWS account info for the integration tests and creates an S3 @@ -73,6 +75,11 @@ public static void setUpForAllIntegTests() throws Exception { System.setProperty("aws.crt.debugnative", "true"); s3 = s3ClientBuilder().build(); s3Async = s3AsyncClientBuilder().build(); + s3NonMultipartAsync = S3AsyncClient.builder() + .credentialsProvider(CREDENTIALS_PROVIDER_CHAIN) + .region(DEFAULT_REGION) + .build(); + s3CrtAsync = S3CrtAsyncClient.builder() .credentialsProvider(CREDENTIALS_PROVIDER_CHAIN) .region(DEFAULT_REGION) @@ -84,6 +91,9 @@ public static void setUpForAllIntegTests() throws Exception { tmJava = S3TransferManager.builder() .s3Client(s3Async) .build(); + tmNonMultipartJava = S3TransferManager.builder() + .s3Client(s3NonMultipartAsync) + .build(); } @@ -91,8 +101,11 @@ public static void setUpForAllIntegTests() throws Exception { public static void cleanUpForAllIntegTests() { s3.close(); s3Async.close(); + s3NonMultipartAsync.close(); s3CrtAsync.close(); tmCrt.close(); + tmJava.close(); + tmNonMultipartJava.close(); CrtResource.waitForNoResources(); } @@ -188,4 +201,11 @@ static Stream transferManagers() { Arguments.of(tmJava)); } + static Stream presignedUrlTransferManagers() { + return Stream.of( + // TODO: uncomment when CRT for presigned URL is supported + // Arguments.of(tmCrt), + Arguments.of(tmNonMultipartJava), + Arguments.of(tmJava)); + } } diff --git a/services-custom/s3-transfer-manager/src/it/java/software/amazon/awssdk/transfer/s3/S3TransferManagerPresignedUrlDownloadIntegrationTest.java b/services-custom/s3-transfer-manager/src/it/java/software/amazon/awssdk/transfer/s3/S3TransferManagerPresignedUrlDownloadIntegrationTest.java new file mode 100644 index 000000000000..003fddb5d5b7 --- /dev/null +++ b/services-custom/s3-transfer-manager/src/it/java/software/amazon/awssdk/transfer/s3/S3TransferManagerPresignedUrlDownloadIntegrationTest.java @@ -0,0 +1,211 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.transfer.s3; + +import static org.assertj.core.api.Assertions.assertThat; +import static software.amazon.awssdk.testutils.service.S3BucketUtils.temporaryBucketName; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Duration; +import java.util.stream.Stream; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import software.amazon.awssdk.core.ResponseBytes; +import software.amazon.awssdk.core.async.AsyncResponseTransformer; +import software.amazon.awssdk.services.s3.model.GetObjectRequest; +import software.amazon.awssdk.services.s3.model.GetObjectResponse; +import software.amazon.awssdk.services.s3.model.PutObjectRequest; +import software.amazon.awssdk.services.s3.presigner.S3Presigner; +import software.amazon.awssdk.services.s3.presigner.model.GetObjectPresignRequest; +import software.amazon.awssdk.services.s3.presigner.model.PresignedGetObjectRequest; +import software.amazon.awssdk.services.s3.presignedurl.model.PresignedUrlDownloadRequest; +import software.amazon.awssdk.testutils.RandomTempFile; +import software.amazon.awssdk.transfer.s3.model.CompletedDownload; +import software.amazon.awssdk.transfer.s3.model.CompletedFileDownload; +import software.amazon.awssdk.transfer.s3.model.PresignedFileDownload; +import software.amazon.awssdk.transfer.s3.model.Download; +import software.amazon.awssdk.transfer.s3.model.PresignedDownloadFileRequest; +import software.amazon.awssdk.transfer.s3.model.PresignedDownloadRequest; +import software.amazon.awssdk.transfer.s3.progress.LoggingTransferListener; +import software.amazon.awssdk.utils.Md5Utils; + +public class S3TransferManagerPresignedUrlDownloadIntegrationTest extends S3IntegrationTestBase { + private static final String BUCKET = temporaryBucketName(S3TransferManagerPresignedUrlDownloadIntegrationTest.class); + private static final String SMALL_KEY = "small-key"; + private static final String LARGE_KEY = "large-key"; + private static final int SMALL_OBJ_SIZE = 5 * 1024 * 1024; + private static final int LARGE_OBJ_SIZE = 16 * 1024 * 1024; + + private static File smallFile; + private static File largeFile; + private static S3Presigner presigner; + + @BeforeAll + public static void setup() throws IOException { + createBucket(BUCKET); + smallFile = new RandomTempFile(SMALL_OBJ_SIZE); + largeFile = new RandomTempFile(LARGE_OBJ_SIZE); + s3.putObject(PutObjectRequest.builder().bucket(BUCKET).key(SMALL_KEY).build(), smallFile.toPath()); + s3.putObject(PutObjectRequest.builder().bucket(BUCKET).key(LARGE_KEY).build(), largeFile.toPath()); + presigner = S3Presigner.builder() + .region(DEFAULT_REGION) + .credentialsProvider(CREDENTIALS_PROVIDER_CHAIN) + .build(); + } + + @AfterAll + public static void cleanup() { + if (presigner != null) { + presigner.close(); + } + deleteBucketAndAllContents(BUCKET); + } + + static Stream testCases() { + return presignedUrlTransferManagers().flatMap(tmArg -> Stream.of( + Arguments.of(tmArg.get()[0], SMALL_KEY, smallFile, SMALL_OBJ_SIZE), + Arguments.of(tmArg.get()[0], LARGE_KEY, largeFile, LARGE_OBJ_SIZE) + )); + } + + @ParameterizedTest(name = "downloadFileWithPresignedUrl_{1}") + @MethodSource("testCases") + void downloadFileWithPresignedUrl_shouldDownloadCorrectly(S3TransferManager tm, String key, + File sourceFile, int objSize) throws Exception { + Path downloadPath = RandomTempFile.randomUncreatedFile().toPath(); + PresignedFileDownload download = tm.downloadFileWithPresignedUrl(createFileDownloadRequest(key, downloadPath)); + CompletedFileDownload completed = download.completionFuture().join(); + + assertThat(Files.exists(downloadPath)).isTrue(); + assertThat(Md5Utils.md5AsBase64(downloadPath.toFile())).isEqualTo(Md5Utils.md5AsBase64(sourceFile)); + assertThat(completed.response().responseMetadata().requestId()).isNotNull(); + } + + @ParameterizedTest(name = "downloadWithPresignedUrl_toBytes_{1}") + @MethodSource("testCases") + void downloadWithPresignedUrl_toBytes_shouldReturnCorrectData(S3TransferManager tm, String key, + File sourceFile, int objSize) { + CompletedDownload> completed = + tm.downloadWithPresignedUrl(createBytesDownloadRequest(key)).completionFuture().join(); + + assertThat(completed.result().asByteArray()).hasSize(objSize); + } + + static Stream progressTestCases() { + return Stream.of( + Arguments.of("multipart", tmJava, LARGE_KEY, null, LARGE_OBJ_SIZE), + Arguments.of("multipart", tmJava, LARGE_KEY, "bytes=0-1048575", 1048576), + Arguments.of("nonMultipart", tmNonMultipartJava, LARGE_KEY, null, LARGE_OBJ_SIZE), + Arguments.of("nonMultipart", tmNonMultipartJava, LARGE_KEY, "bytes=0-1048575", 1048576) + ); + } + + @ParameterizedTest(name = "downloadFileWithPresignedUrl_progress_{0}_range={3}") + @MethodSource("progressTestCases") + void downloadFileWithPresignedUrl_progressTracking(String tmType, S3TransferManager tm, String key, + String range, int expectedSize) throws Exception { + Path downloadPath = RandomTempFile.randomUncreatedFile().toPath(); + + PresignedUrlDownloadRequest.Builder requestBuilder = PresignedUrlDownloadRequest.builder() + .presignedUrl(createPresignedRequest(key).url()); + if (range != null) { + requestBuilder.range(range); + } + + PresignedFileDownload download = tm.downloadFileWithPresignedUrl( + PresignedDownloadFileRequest.builder() + .presignedUrlDownloadRequest(requestBuilder.build()) + .destination(downloadPath) + .addTransferListener(LoggingTransferListener.create()) + .build()); + + download.completionFuture().join(); + + // Verify progress tracking worked - totalBytes is set correctly + assertThat(download.progress().snapshot().totalBytes()).isPresent(); + assertThat(download.progress().snapshot().totalBytes().getAsLong()).isEqualTo(expectedSize); + + // Verify transferredBytes reached expectedSize + assertThat(download.progress().snapshot().transferredBytes()).isEqualTo(expectedSize); + + // Verify file size matches expected + assertThat(downloadPath.toFile().length()).isEqualTo(expectedSize); + } + + @ParameterizedTest(name = "downloadWithPresignedUrl_toBytes_progress_{0}_range={3}") + @MethodSource("progressTestCases") + void downloadWithPresignedUrl_toBytes_progressTracking(String tmType, S3TransferManager tm, String key, + String range, int expectedSize) throws Exception { + + PresignedUrlDownloadRequest.Builder requestBuilder = PresignedUrlDownloadRequest.builder() + .presignedUrl(createPresignedRequest(key).url()); + if (range != null) { + requestBuilder.range(range); + } + + Download> download = tm.downloadWithPresignedUrl( + PresignedDownloadRequest.>builder() + .presignedUrlDownloadRequest(requestBuilder.build()) + .responseTransformer(AsyncResponseTransformer.toBytes()) + .addTransferListener(LoggingTransferListener.create()) + .build()); + + CompletedDownload> completed = download.completionFuture().join(); + + assertThat(download.progress().snapshot().totalBytes()).isPresent(); + assertThat(download.progress().snapshot().totalBytes().getAsLong()).isEqualTo(expectedSize); + + assertThat(download.progress().snapshot().transferredBytes()).isEqualTo(expectedSize); + + assertThat(completed.result().asByteArray()).hasSize(expectedSize); + } + + private static PresignedDownloadFileRequest createFileDownloadRequest(String key, Path destination) { + return PresignedDownloadFileRequest.builder() + .presignedUrlDownloadRequest(PresignedUrlDownloadRequest.builder() + .presignedUrl(createPresignedRequest(key).url()) + .build()) + .destination(destination) + .addTransferListener(LoggingTransferListener.create()) + .build(); + } + + private static PresignedDownloadRequest> createBytesDownloadRequest(String key) { + return PresignedDownloadRequest.builder() + .presignedUrlDownloadRequest(PresignedUrlDownloadRequest.builder() + .presignedUrl(createPresignedRequest(key).url()) + .build()) + .responseTransformer(AsyncResponseTransformer.toBytes()) + .addTransferListener(LoggingTransferListener.create()) + .build(); + } + + private static PresignedGetObjectRequest createPresignedRequest(String key) { + return presigner.presignGetObject(GetObjectPresignRequest.builder() + .signatureDuration(Duration.ofMinutes(10)) + .getObjectRequest(GetObjectRequest.builder() + .bucket(BUCKET) + .key(key) + .build()) + .build()); + } +} diff --git a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/S3TransferManager.java b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/S3TransferManager.java index 6bdc2c12d580..b42582395a19 100644 --- a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/S3TransferManager.java +++ b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/S3TransferManager.java @@ -43,6 +43,9 @@ import software.amazon.awssdk.transfer.s3.model.DownloadRequest; import software.amazon.awssdk.transfer.s3.model.FileDownload; import software.amazon.awssdk.transfer.s3.model.FileUpload; +import software.amazon.awssdk.transfer.s3.model.PresignedDownloadFileRequest; +import software.amazon.awssdk.transfer.s3.model.PresignedDownloadRequest; +import software.amazon.awssdk.transfer.s3.model.PresignedFileDownload; import software.amazon.awssdk.transfer.s3.model.ResumableFileDownload; import software.amazon.awssdk.transfer.s3.model.ResumableFileUpload; import software.amazon.awssdk.transfer.s3.model.Upload; @@ -701,6 +704,107 @@ default Copy copy(Consumer copyRequestBuilder) { return copy(CopyRequest.builder().applyMutation(copyRequestBuilder).build()); } + /** + * Downloads an object using a pre-signed URL to a local file. For non-file-based downloads, you may use + * {@link #downloadWithPresignedUrl(PresignedDownloadRequest)} instead. + *

+ * This method supports multipart downloads when using a CRT-based or multipart-enabled S3 client, + * providing enhanced throughput and reliability for large objects. Progress can be monitored + * through {@link TransferListener}s attached to the request. + *

+ * The SDK will create a new file if the provided destination doesn't exist. If the file already exists, + * it will be replaced. In the event of an error, the SDK will NOT attempt to delete + * the file, leaving it as-is. + *

+ * Note: The result of the operation doesn't support pause and resume functionality. + *

+ *

+ * Usage Example: + * {@snippet : + * S3TransferManager transferManager = S3TransferManager.create(); + * + * // Create presigned URL (typically done by another service) + * PresignedUrlDownloadRequest presignedRequest = PresignedUrlDownloadRequest.builder() + * .presignedUrl(presignedUrl) + * .build(); + * + * PresignedDownloadFileRequest request = PresignedDownloadFileRequest.builder() + * .presignedUrlDownloadRequest(presignedRequest) + * .destination(Paths.get("downloaded-file.txt")) + * .addTransferListener( + * LoggingTransferListener.create()) + * .build(); + * + * PresignedFileDownload download = transferManager.downloadFileWithPresignedUrl(request); + * download.completionFuture().join(); + * } + * + * @param presignedDownloadFileRequest the presigned download file request + * @return A {@link PresignedFileDownload} that can be used to track the ongoing transfer + * @see #downloadFileWithPresignedUrl(Consumer) + * @see #downloadWithPresignedUrl(PresignedDownloadRequest) + */ + default PresignedFileDownload downloadFileWithPresignedUrl(PresignedDownloadFileRequest presignedDownloadFileRequest) { + throw new UnsupportedOperationException(); + } + + /** + * This is a convenience method that creates an instance of the {@link PresignedDownloadFileRequest} builder, + * avoiding the need to create one manually via {@link PresignedDownloadFileRequest#builder()}. + *

+ * Note: The result of the operation doesn't support pause and resume functionality. + *

+ * + * @see #downloadFileWithPresignedUrl(PresignedDownloadFileRequest) + */ + default PresignedFileDownload downloadFileWithPresignedUrl( + Consumer presignedDownloadFileRequest) { + return downloadFileWithPresignedUrl( + PresignedDownloadFileRequest.builder().applyMutation(presignedDownloadFileRequest).build()); + } + + /** + * Downloads an object using a pre-signed URL through the given {@link AsyncResponseTransformer}. For downloading + * to a file, you may use {@link #downloadFileWithPresignedUrl(PresignedDownloadFileRequest)} instead. + *

+ * This method supports multipart downloads when using a CRT-based or multipart-enabled S3 client, + * providing enhanced throughput and reliability for large objects. Progress can be monitored + * through {@link TransferListener}s attached to the request. + *

+ * Note: The result of the operation doesn't support pause and resume functionality. + *

+ *

+ * Usage Example (downloading to memory - not suitable for large objects): + * {@snippet : + * S3TransferManager transferManager = S3TransferManager.create(); + * + * // Create presigned URL (typically done by another service) + * PresignedUrlDownloadRequest presignedRequest = PresignedUrlDownloadRequest.builder() + * .presignedUrl(presignedUrl) + * .build(); + * + * PresignedDownloadRequest> request = + * PresignedDownloadRequest.builder() + * .presignedUrlDownloadRequest(presignedRequest) + * .responseTransformer(AsyncResponseTransformer.toBytes()) + * .addTransferListener(LoggingTransferListener.create()) + * .build(); + * + * Download> download = transferManager.downloadWithPresignedUrl(request); + * ResponseBytes result = download.completionFuture().join().result(); + * } + * + * @param presignedDownloadRequest the presigned download request + * @param The type of data the {@link AsyncResponseTransformer} produces + * @return A {@link Download} that can be used to track the ongoing transfer + * @see #downloadFileWithPresignedUrl(PresignedDownloadFileRequest) + * @see AsyncResponseTransformer + */ + default Download downloadWithPresignedUrl( + PresignedDownloadRequest presignedDownloadRequest) { + throw new UnsupportedOperationException(); + } + /** * Create an {@code S3TransferManager} using the default values. *

diff --git a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/DelegatingS3TransferManager.java b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/DelegatingS3TransferManager.java index f2929bf8f988..6907f1f00d08 100644 --- a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/DelegatingS3TransferManager.java +++ b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/DelegatingS3TransferManager.java @@ -27,6 +27,9 @@ import software.amazon.awssdk.transfer.s3.model.DownloadRequest; import software.amazon.awssdk.transfer.s3.model.FileDownload; import software.amazon.awssdk.transfer.s3.model.FileUpload; +import software.amazon.awssdk.transfer.s3.model.PresignedDownloadFileRequest; +import software.amazon.awssdk.transfer.s3.model.PresignedDownloadRequest; +import software.amazon.awssdk.transfer.s3.model.PresignedFileDownload; import software.amazon.awssdk.transfer.s3.model.ResumableFileDownload; import software.amazon.awssdk.transfer.s3.model.Upload; import software.amazon.awssdk.transfer.s3.model.UploadDirectoryRequest; @@ -85,6 +88,16 @@ public Copy copy(CopyRequest copyRequest) { return delegate.copy(copyRequest); } + @Override + public PresignedFileDownload downloadFileWithPresignedUrl(PresignedDownloadFileRequest presignedDownloadFileRequest) { + return delegate.downloadFileWithPresignedUrl(presignedDownloadFileRequest); + } + + @Override + public Download downloadWithPresignedUrl(PresignedDownloadRequest presignedDownloadRequest) { + return delegate.downloadWithPresignedUrl(presignedDownloadRequest); + } + @Override public void close() { delegate.close(); diff --git a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/GenericS3TransferManager.java b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/GenericS3TransferManager.java index 4814b862e379..f92c5196a095 100644 --- a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/GenericS3TransferManager.java +++ b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/GenericS3TransferManager.java @@ -55,6 +55,7 @@ import software.amazon.awssdk.transfer.s3.internal.model.DefaultDownload; import software.amazon.awssdk.transfer.s3.internal.model.DefaultFileDownload; import software.amazon.awssdk.transfer.s3.internal.model.DefaultFileUpload; +import software.amazon.awssdk.transfer.s3.internal.model.DefaultPresignedFileDownload; import software.amazon.awssdk.transfer.s3.internal.model.DefaultUpload; import software.amazon.awssdk.transfer.s3.internal.progress.DefaultTransferProgress; import software.amazon.awssdk.transfer.s3.internal.progress.DefaultTransferProgressSnapshot; @@ -75,6 +76,9 @@ import software.amazon.awssdk.transfer.s3.model.DownloadRequest; import software.amazon.awssdk.transfer.s3.model.FileDownload; import software.amazon.awssdk.transfer.s3.model.FileUpload; +import software.amazon.awssdk.transfer.s3.model.PresignedDownloadFileRequest; +import software.amazon.awssdk.transfer.s3.model.PresignedDownloadRequest; +import software.amazon.awssdk.transfer.s3.model.PresignedFileDownload; import software.amazon.awssdk.transfer.s3.model.ResumableFileDownload; import software.amazon.awssdk.transfer.s3.model.ResumableFileUpload; import software.amazon.awssdk.transfer.s3.model.Upload; @@ -330,7 +334,7 @@ private CopyObjectRequest attachSdkAttribute(CopyObjectRequest copyObjectRequest } private GetObjectRequest attachSdkAttribute(GetObjectRequest request, - Consumer builderMutation) { + Consumer builderMutation) { AwsRequestOverrideConfiguration modifiedRequestOverrideConfig = request.overrideConfiguration() .map(o -> o.toBuilder().applyMutation(builderMutation).build()) @@ -597,6 +601,78 @@ public final Copy copy(CopyRequest copyRequest) { return new DefaultCopy(returnFuture, progressUpdater.progress()); } + @Override + public final PresignedFileDownload downloadFileWithPresignedUrl(PresignedDownloadFileRequest presignedDownloadFileRequest) { + Validate.paramNotNull(presignedDownloadFileRequest, "presignedDownloadFileRequest"); + + AsyncResponseTransformer responseTransformer = + AsyncResponseTransformer.toFile(presignedDownloadFileRequest.destination(), + FileTransformerConfiguration.defaultCreateOrReplaceExisting()); + + CompletableFuture returnFuture = new CompletableFuture<>(); + + TransferProgressUpdater progressUpdater = new TransferProgressUpdater(presignedDownloadFileRequest, null); + progressUpdater.transferInitiated(); + + responseTransformer = isS3ClientMultipartEnabled() + && presignedDownloadFileRequest.presignedUrlDownloadRequest().range() == null + ? progressUpdater.wrapForNonSerialFileDownload( + responseTransformer, GetObjectRequest.builder().build()) + : progressUpdater.wrapResponseTransformer(responseTransformer); + progressUpdater.registerCompletion(returnFuture); + + try { + CompletableFuture future = s3AsyncClient.presignedUrlExtension().getObject( + presignedDownloadFileRequest.presignedUrlDownloadRequest(), responseTransformer); + + CompletableFutureUtils.forwardExceptionTo(returnFuture, future); + CompletableFutureUtils.forwardTransformedResultTo(future, returnFuture, + res -> CompletedFileDownload.builder() + .response(res) + .build()); + } catch (Throwable throwable) { + returnFuture.completeExceptionally(throwable); + } + + return new DefaultPresignedFileDownload(returnFuture, progressUpdater.progress()); + } + + @Override + public final Download downloadWithPresignedUrl( + PresignedDownloadRequest presignedDownloadRequest) { + Validate.paramNotNull(presignedDownloadRequest, "presignedDownloadRequest"); + + AsyncResponseTransformer responseTransformer = + presignedDownloadRequest.responseTransformer(); + + CompletableFuture> returnFuture = new CompletableFuture<>(); + + TransferProgressUpdater progressUpdater = new TransferProgressUpdater(presignedDownloadRequest, null); + progressUpdater.transferInitiated(); + + responseTransformer = isS3ClientMultipartEnabled() + && presignedDownloadRequest.presignedUrlDownloadRequest().range() == null + ? progressUpdater.wrapForNonSerialFileDownload( + responseTransformer, GetObjectRequest.builder().build()) + : progressUpdater.wrapResponseTransformer(responseTransformer); + progressUpdater.registerCompletion(returnFuture); + + try { + CompletableFuture future = s3AsyncClient.presignedUrlExtension().getObject( + presignedDownloadRequest.presignedUrlDownloadRequest(), responseTransformer); + + CompletableFutureUtils.forwardExceptionTo(returnFuture, future); + CompletableFutureUtils.forwardTransformedResultTo(future, returnFuture, + r -> CompletedDownload.builder() + .result(r) + .build()); + } catch (Throwable throwable) { + returnFuture.completeExceptionally(throwable); + } + + return new DefaultDownload<>(returnFuture, progressUpdater.progress()); + } + @Override public final void close() { if (isDefaultS3AsyncClient) { diff --git a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/model/DefaultPresignedFileDownload.java b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/model/DefaultPresignedFileDownload.java new file mode 100644 index 000000000000..253c60a8a42a --- /dev/null +++ b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/model/DefaultPresignedFileDownload.java @@ -0,0 +1,79 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.transfer.s3.internal.model; + +import java.util.Objects; +import java.util.concurrent.CompletableFuture; +import software.amazon.awssdk.annotations.SdkInternalApi; +import software.amazon.awssdk.transfer.s3.model.CompletedFileDownload; +import software.amazon.awssdk.transfer.s3.model.PresignedFileDownload; +import software.amazon.awssdk.transfer.s3.progress.TransferProgress; +import software.amazon.awssdk.utils.ToString; +import software.amazon.awssdk.utils.Validate; + +@SdkInternalApi +public final class DefaultPresignedFileDownload implements PresignedFileDownload { + private final CompletableFuture completionFuture; + private final TransferProgress progress; + + public DefaultPresignedFileDownload(CompletableFuture completionFuture, + TransferProgress progress) { + this.completionFuture = Validate.paramNotNull(completionFuture, "completionFuture"); + this.progress = Validate.paramNotNull(progress, "progress"); + } + + @Override + public CompletableFuture completionFuture() { + return completionFuture; + } + + @Override + public TransferProgress progress() { + return progress; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + + DefaultPresignedFileDownload that = (DefaultPresignedFileDownload) o; + + if (!Objects.equals(completionFuture, that.completionFuture)) { + return false; + } + return Objects.equals(progress, that.progress); + } + + @Override + public int hashCode() { + int result = completionFuture != null ? completionFuture.hashCode() : 0; + result = 31 * result + (progress != null ? progress.hashCode() : 0); + return result; + } + + @Override + public String toString() { + return ToString.builder("DefaultPresignedFileDownload") + .add("completionFuture", completionFuture) + .add("progress", progress) + .build(); + } +} diff --git a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/model/PresignedDownloadFileRequest.java b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/model/PresignedDownloadFileRequest.java new file mode 100644 index 000000000000..b165292b22e5 --- /dev/null +++ b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/model/PresignedDownloadFileRequest.java @@ -0,0 +1,286 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.transfer.s3.model; + +import java.io.File; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.Objects; +import java.util.function.Consumer; +import software.amazon.awssdk.annotations.NotThreadSafe; +import software.amazon.awssdk.annotations.SdkPublicApi; +import software.amazon.awssdk.services.s3.presignedurl.model.PresignedUrlDownloadRequest; +import software.amazon.awssdk.transfer.s3.S3TransferManager; +import software.amazon.awssdk.transfer.s3.progress.TransferListener; +import software.amazon.awssdk.utils.ToString; +import software.amazon.awssdk.utils.Validate; +import software.amazon.awssdk.utils.builder.CopyableBuilder; +import software.amazon.awssdk.utils.builder.ToCopyableBuilder; + +/** + * Download an object using a pre-signed URL to a local file. For non-file-based downloads, you may use {@link + * PresignedDownloadRequest} instead. + * + * @see S3TransferManager#downloadFileWithPresignedUrl(PresignedDownloadFileRequest) + */ +@SdkPublicApi +public final class PresignedDownloadFileRequest + implements TransferObjectRequest, ToCopyableBuilder { + + private final Path destination; + private final PresignedUrlDownloadRequest presignedUrlDownloadRequest; + private final List transferListeners; + + private PresignedDownloadFileRequest(DefaultBuilder builder) { + this.destination = Validate.paramNotNull(builder.destination, "destination"); + this.presignedUrlDownloadRequest = Validate.paramNotNull(builder.presignedUrlDownloadRequest, + "presignedUrlDownloadRequest"); + this.transferListeners = builder.transferListeners; + } + + /** + * Creates a builder that can be used to create a {@link PresignedDownloadFileRequest}. + * + * @see S3TransferManager#downloadFileWithPresignedUrl(PresignedDownloadFileRequest) + */ + public static Builder builder() { + return new DefaultBuilder(); + } + + @Override + public Builder toBuilder() { + return new DefaultBuilder(this); + } + + /** + * The {@link Path} to file that response contents will be written to. The file must not exist or this method + * will throw an exception. If the file is not writable by the current user then an exception will be thrown. + * + * @return the destination path + */ + public Path destination() { + return destination; + } + + /** + * @return The {@link PresignedUrlDownloadRequest} request that should be used for the download + */ + public PresignedUrlDownloadRequest presignedUrlDownloadRequest() { + return presignedUrlDownloadRequest; + } + + /** + * + * @return List of {@link TransferListener}s that will be notified as part of this request. + */ + @Override + public List transferListeners() { + return transferListeners; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + + PresignedDownloadFileRequest that = (PresignedDownloadFileRequest) o; + + if (!Objects.equals(destination, that.destination)) { + return false; + } + if (!Objects.equals(presignedUrlDownloadRequest, that.presignedUrlDownloadRequest)) { + return false; + } + return Objects.equals(transferListeners, that.transferListeners); + } + + @Override + public int hashCode() { + int result = destination != null ? destination.hashCode() : 0; + result = 31 * result + (presignedUrlDownloadRequest != null ? presignedUrlDownloadRequest.hashCode() : 0); + result = 31 * result + (transferListeners != null ? transferListeners.hashCode() : 0); + return result; + } + + @Override + public String toString() { + return ToString.builder("PresignedDownloadFileRequest") + .add("destination", destination) + .add("presignedUrlDownloadRequest", presignedUrlDownloadRequest) + .add("transferListeners", transferListeners) + .build(); + } + + public static Class serializableBuilderClass() { + return DefaultBuilder.class; + } + + /** + * A builder for a {@link PresignedDownloadFileRequest}, created with {@link #builder()} + */ + @SdkPublicApi + @NotThreadSafe + public interface Builder extends CopyableBuilder { + + /** + * The {@link Path} to file that response contents will be written to. The file must not exist or this method + * will throw an exception. If the file is not writable by the current user then an exception will be thrown. + * + * @param destination the destination path + * @return Returns a reference to this object so that method calls can be chained together. + */ + Builder destination(Path destination); + + /** + * The file that response contents will be written to. The file must not exist or this method + * will throw an exception. If the file is not writable by the current user then an exception will be thrown. + * + * @param destination the destination path + * @return Returns a reference to this object so that method calls can be chained together. + */ + default Builder destination(File destination) { + Validate.paramNotNull(destination, "destination"); + return destination(destination.toPath()); + } + + /** + * The {@link PresignedUrlDownloadRequest} request that should be used for the download + * + * @param presignedUrlDownloadRequest the presigned URL download request + * @return a reference to this object so that method calls can be chained together. + * @see #presignedUrlDownloadRequest(Consumer) + */ + Builder presignedUrlDownloadRequest(PresignedUrlDownloadRequest presignedUrlDownloadRequest); + + /** + * The {@link PresignedUrlDownloadRequest} request that should be used for the download + * + *

+ * This is a convenience method that creates an instance of the {@link PresignedUrlDownloadRequest} builder avoiding the + * need to create one manually via {@link PresignedUrlDownloadRequest#builder()}. + * + * @param presignedUrlDownloadRequestBuilder the presigned URL download request + * @return a reference to this object so that method calls can be chained together. + * @see #presignedUrlDownloadRequest(PresignedUrlDownloadRequest) + */ + default Builder presignedUrlDownloadRequest( + Consumer presignedUrlDownloadRequestBuilder) { + PresignedUrlDownloadRequest request = PresignedUrlDownloadRequest.builder() + .applyMutation(presignedUrlDownloadRequestBuilder) + .build(); + presignedUrlDownloadRequest(request); + return this; + } + + /** + * The {@link TransferListener}s that will be notified as part of this request. This method overrides and replaces any + * transferListeners that have already been set. Add an optional request override configuration. + * + * @param transferListeners the collection of transferListeners + * @return Returns a reference to this object so that method calls can be chained together. + * @return This builder for method chaining. + * @see TransferListener + */ + Builder transferListeners(Collection transferListeners); + + /** + * Add a {@link TransferListener} that will be notified as part of this request. + * + * @param transferListener the transferListener to add + * @return Returns a reference to this object so that method calls can be chained together. + * @see TransferListener + */ + Builder addTransferListener(TransferListener transferListener); + + } + + private static final class DefaultBuilder implements Builder { + private Path destination; + private PresignedUrlDownloadRequest presignedUrlDownloadRequest; + private List transferListeners; + + private DefaultBuilder() { + } + + private DefaultBuilder(PresignedDownloadFileRequest presignedDownloadFileRequest) { + this.destination = presignedDownloadFileRequest.destination; + this.presignedUrlDownloadRequest = presignedDownloadFileRequest.presignedUrlDownloadRequest; + this.transferListeners = presignedDownloadFileRequest.transferListeners; + } + + @Override + public Builder destination(Path destination) { + this.destination = Validate.paramNotNull(destination, "destination"); + return this; + } + + public Path getDestination() { + return destination; + } + + public void setDestination(Path destination) { + destination(destination); + } + + @Override + public DefaultBuilder presignedUrlDownloadRequest(PresignedUrlDownloadRequest presignedUrlDownloadRequest) { + this.presignedUrlDownloadRequest = presignedUrlDownloadRequest; + return this; + } + + public PresignedUrlDownloadRequest getPresignedUrlDownloadRequest() { + return presignedUrlDownloadRequest; + } + + public void setPresignedUrlDownloadRequest(PresignedUrlDownloadRequest presignedUrlDownloadRequest) { + presignedUrlDownloadRequest(presignedUrlDownloadRequest); + } + + @Override + public DefaultBuilder transferListeners(Collection transferListeners) { + this.transferListeners = transferListeners != null ? new ArrayList<>(transferListeners) : null; + return this; + } + + @Override + public Builder addTransferListener(TransferListener transferListener) { + if (transferListeners == null) { + transferListeners = new ArrayList<>(); + } + transferListeners.add(transferListener); + return this; + } + + public List getTransferListeners() { + return transferListeners; + } + + public void setTransferListeners(Collection transferListeners) { + transferListeners(transferListeners); + } + + @Override + public PresignedDownloadFileRequest build() { + return new PresignedDownloadFileRequest(this); + } + } +} \ No newline at end of file diff --git a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/model/PresignedDownloadRequest.java b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/model/PresignedDownloadRequest.java new file mode 100644 index 000000000000..1a45eeb21254 --- /dev/null +++ b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/model/PresignedDownloadRequest.java @@ -0,0 +1,391 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.transfer.s3.model; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.Objects; +import java.util.function.Consumer; +import software.amazon.awssdk.annotations.SdkPublicApi; +import software.amazon.awssdk.core.async.AsyncResponseTransformer; +import software.amazon.awssdk.services.s3.model.GetObjectResponse; +import software.amazon.awssdk.services.s3.presignedurl.model.PresignedUrlDownloadRequest; +import software.amazon.awssdk.transfer.s3.S3TransferManager; +import software.amazon.awssdk.transfer.s3.model.PresignedDownloadRequest.TypedBuilder; +import software.amazon.awssdk.transfer.s3.progress.TransferListener; +import software.amazon.awssdk.utils.ToString; +import software.amazon.awssdk.utils.Validate; +import software.amazon.awssdk.utils.builder.CopyableBuilder; +import software.amazon.awssdk.utils.builder.ToCopyableBuilder; + +/** + * Represents the request to download an object using a pre-signed URL through the given + * {@link AsyncResponseTransformer}. For downloading to a file, + * you may use {@link PresignedDownloadFileRequest} instead. + * + * @see S3TransferManager#downloadWithPresignedUrl(PresignedDownloadRequest) + */ +@SdkPublicApi +public final class PresignedDownloadRequest + implements TransferObjectRequest, + ToCopyableBuilder, PresignedDownloadRequest> { + + private final AsyncResponseTransformer responseTransformer; + private final PresignedUrlDownloadRequest presignedUrlDownloadRequest; + private final List transferListeners; + + private PresignedDownloadRequest(DefaultTypedBuilder builder) { + this.responseTransformer = Validate.paramNotNull(builder.responseTransformer, "responseTransformer"); + this.presignedUrlDownloadRequest = Validate.paramNotNull(builder.presignedUrlDownloadRequest, + "presignedUrlDownloadRequest"); + this.transferListeners = builder.transferListeners; + } + + /** + * Creates a builder that can be used to create a {@link PresignedDownloadRequest}. + * + * @see UntypedBuilder + */ + public static UntypedBuilder builder() { + return new DefaultUntypedBuilder(); + } + + + @Override + public TypedBuilder toBuilder() { + return new DefaultTypedBuilder<>(this); + } + + /** + * The {@link AsyncResponseTransformer} that response contents will be written to. + * + * @return the response transformer + */ + public AsyncResponseTransformer responseTransformer() { + return responseTransformer; + } + + /** + * @return The {@link PresignedUrlDownloadRequest} request that should be used for the download + */ + public PresignedUrlDownloadRequest presignedUrlDownloadRequest() { + return presignedUrlDownloadRequest; + } + + /** + * @return the List of transferListeners. + * @see TypedBuilder#transferListeners(Collection) + */ + @Override + public List transferListeners() { + return transferListeners; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + + PresignedDownloadRequest that = (PresignedDownloadRequest) o; + + if (!Objects.equals(responseTransformer, that.responseTransformer)) { + return false; + } + if (!Objects.equals(presignedUrlDownloadRequest, that.presignedUrlDownloadRequest)) { + return false; + } + return Objects.equals(transferListeners, that.transferListeners); + } + + @Override + public int hashCode() { + int result = responseTransformer != null ? responseTransformer.hashCode() : 0; + result = 31 * result + (presignedUrlDownloadRequest != null ? presignedUrlDownloadRequest.hashCode() : 0); + result = 31 * result + (transferListeners != null ? transferListeners.hashCode() : 0); + return result; + } + + @Override + public String toString() { + return ToString.builder("PresignedDownloadRequest") + .add("responseTransformer", responseTransformer) + .add("presignedUrlDownloadRequest", presignedUrlDownloadRequest) + .add("transferListeners", transferListeners) + .build(); + } + + /** + * Initial calls to {@link PresignedDownloadRequest#builder()} return an {@link UntypedBuilder}, where the builder is not yet + * parameterized with the generic type associated with {@link PresignedDownloadRequest}. + * This prevents the otherwise awkward syntax of having to explicitly cast the builder type, e.g., + *

+     * {@code PresignedDownloadRequest.>builder()}
+     * 
+ * Instead, the type may be inferred as part of specifying the {@link #responseTransformer(AsyncResponseTransformer)} + * parameter, at which point the builder chain will return a new {@link TypedBuilder}. + */ + public interface UntypedBuilder { + + /** + * The {@link PresignedUrlDownloadRequest} request that should be used for the download + * + * @param presignedUrlDownloadRequest the presigned URL download request + * @return a reference to this object so that method calls can be chained together. + * @see #presignedUrlDownloadRequest(Consumer) + */ + UntypedBuilder presignedUrlDownloadRequest(PresignedUrlDownloadRequest presignedUrlDownloadRequest); + + /** + * The {@link PresignedUrlDownloadRequest} request that should be used for the download + *

+ * This is a convenience method that creates an instance of the {@link PresignedUrlDownloadRequest} + * builder, avoiding the need to create one manually via {@link PresignedUrlDownloadRequest#builder()}. + * + * @param presignedUrlDownloadRequestBuilder the presigned URL download request + * @return a reference to this object so that method calls can be chained together. + * @see #presignedUrlDownloadRequest(PresignedUrlDownloadRequest) + */ + default UntypedBuilder presignedUrlDownloadRequest( + Consumer presignedUrlDownloadRequestBuilder) { + PresignedUrlDownloadRequest request = PresignedUrlDownloadRequest.builder() + .applyMutation(presignedUrlDownloadRequestBuilder) + .build(); + presignedUrlDownloadRequest(request); + return this; + } + + /** + * The {@link TransferListener}s that will be notified as part of this request. This method + * overrides and replaces any transferListeners that have already been set. Add an optional + * request override configuration. + * + * @param transferListeners the collection of transferListeners + * @return Returns a reference to this object so that method calls can be chained together. + * @return This builder for method chaining. + * @see TransferListener + */ + UntypedBuilder transferListeners( + Collection transferListeners); + + /** + * Adds a {@link TransferListener} that will be notified as part of this request. + * + * @param transferListener the transferListener to add + * @return Returns a reference to this object so that method calls can be chained together. + * @see TransferListener + */ + UntypedBuilder addTransferListener(TransferListener transferListener); + + /** + * Specifies the {@link AsyncResponseTransformer} that should be used for the download. This + * method also infers the generic type of {@link PresignedDownloadRequest} to create, + * inferred from the second type parameter of the provided {@link AsyncResponseTransformer}. + * E.g, specifying {@link AsyncResponseTransformer#toBytes()} would result in inferring the + * type of the {@link PresignedDownloadRequest} to be of {@code ResponseBytes}. + * See the static factory methods available in {@link AsyncResponseTransformer}. + * + * @param responseTransformer the AsyncResponseTransformer + * @param the type of {@link PresignedDownloadRequest} to create + * @return a reference to this object so that method calls can be chained together. + * @see AsyncResponseTransformer + */ + TypedBuilder responseTransformer( + AsyncResponseTransformer responseTransformer); + } + + private static final class DefaultUntypedBuilder implements UntypedBuilder { + private PresignedUrlDownloadRequest presignedUrlDownloadRequest; + private List transferListeners; + + private DefaultUntypedBuilder() { + } + + @Override + public UntypedBuilder presignedUrlDownloadRequest(PresignedUrlDownloadRequest presignedUrlDownloadRequest) { + this.presignedUrlDownloadRequest = presignedUrlDownloadRequest; + return this; + } + + @Override + public UntypedBuilder transferListeners( + Collection transferListeners) { + this.transferListeners = transferListeners != null ? + new ArrayList<>(transferListeners) : null; + return this; + } + + @Override + public UntypedBuilder addTransferListener(TransferListener transferListener) { + if (transferListeners == null) { + transferListeners = new ArrayList<>(); + } + transferListeners.add(transferListener); + return this; + } + + public List getTransferListeners() { + return transferListeners; + } + + public void setTransferListeners( + Collection transferListeners) { + transferListeners(transferListeners); + } + + @Override + public TypedBuilder responseTransformer( + AsyncResponseTransformer responseTransformer) { + return new DefaultTypedBuilder() + .presignedUrlDownloadRequest(presignedUrlDownloadRequest) + .transferListeners(transferListeners) + .responseTransformer(responseTransformer); + } + } + + /** + * The type-parameterized version of {@link UntypedBuilder}. This builder's type is inferred as part of specifying {@link + * UntypedBuilder#responseTransformer(AsyncResponseTransformer)}, after which this builder can be used to construct a {@link + * PresignedDownloadRequest} with the same generic type. + */ + public interface TypedBuilder extends CopyableBuilder, PresignedDownloadRequest> { + + /** + * The {@link PresignedUrlDownloadRequest} request that should be used for the download + * + * @param presignedUrlDownloadRequest the presigned URL download request + * @return a reference to this object so that method calls can be chained together. + * @see #presignedUrlDownloadRequest(Consumer) + */ + TypedBuilder presignedUrlDownloadRequest(PresignedUrlDownloadRequest presignedUrlDownloadRequest); + + /** + * The {@link PresignedUrlDownloadRequest} request that should be used for the download + *

+ * This is a convenience method that creates an instance of the {@link PresignedUrlDownloadRequest} builder, + * avoiding the need to create one manually via {@link PresignedUrlDownloadRequest#builder()}. + * + * @param presignedUrlDownloadRequestBuilder the presigned URL download request + * @return a reference to this object so that method calls can be chained together. + * @see #presignedUrlDownloadRequest(PresignedUrlDownloadRequest) + */ + default TypedBuilder presignedUrlDownloadRequest( + Consumer presignedUrlDownloadRequestBuilder) { + PresignedUrlDownloadRequest request = PresignedUrlDownloadRequest.builder() + .applyMutation(presignedUrlDownloadRequestBuilder) + .build(); + presignedUrlDownloadRequest(request); + return this; + } + + /** + * The {@link TransferListener}s that will be notified as part of this request. This method overrides and + * replaces any transferListeners that have already been set. Add an optional request override + * configuration. + * + * @param transferListeners the collection of transferListeners + * @return Returns a reference to this object so that method calls can be chained together. + * @return This builder for method chaining. + * @see TransferListener + */ + TypedBuilder transferListeners( + Collection transferListeners); + + /** + * Add a {@link TransferListener} that will be notified as part of this request. + * + * @param transferListener the transferListener to add + * @return Returns a reference to this object so that method calls can be chained together. + * @see TransferListener + */ + TypedBuilder addTransferListener(TransferListener transferListener); + + /** + * Specifies the {@link AsyncResponseTransformer} that should be used for the download. The generic type used is + * constrained by the {@link UntypedBuilder#responseTransformer(AsyncResponseTransformer)} that was previously used to + * create this {@link TypedBuilder}. + * + * @param responseTransformer the AsyncResponseTransformer + * @return a reference to this object so that method calls can be chained together. + * @see AsyncResponseTransformer + */ + TypedBuilder responseTransformer( + AsyncResponseTransformer responseTransformer); + } + + private static class DefaultTypedBuilder implements TypedBuilder { + private PresignedUrlDownloadRequest presignedUrlDownloadRequest; + private List transferListeners; + private AsyncResponseTransformer responseTransformer; + + private DefaultTypedBuilder() { + } + + private DefaultTypedBuilder(PresignedDownloadRequest request) { + this.presignedUrlDownloadRequest = request.presignedUrlDownloadRequest; + this.responseTransformer = request.responseTransformer; + this.transferListeners = request.transferListeners; + } + + @Override + public TypedBuilder presignedUrlDownloadRequest(PresignedUrlDownloadRequest presignedUrlDownloadRequest) { + this.presignedUrlDownloadRequest = presignedUrlDownloadRequest; + return this; + } + + @Override + public TypedBuilder responseTransformer( + AsyncResponseTransformer responseTransformer) { + this.responseTransformer = responseTransformer; + return this; + } + + @Override + public TypedBuilder transferListeners( + Collection transferListeners) { + this.transferListeners = transferListeners != null ? + new ArrayList<>(transferListeners) : null; + return this; + } + + @Override + public TypedBuilder addTransferListener(TransferListener transferListener) { + if (transferListeners == null) { + transferListeners = new ArrayList<>(); + } + transferListeners.add(transferListener); + return this; + } + + public List getTransferListeners() { + return transferListeners; + } + + public void setTransferListeners( + Collection transferListeners) { + transferListeners(transferListeners); + } + + @Override + public PresignedDownloadRequest build() { + return new PresignedDownloadRequest<>(this); + } + } +} \ No newline at end of file diff --git a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/model/PresignedFileDownload.java b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/model/PresignedFileDownload.java new file mode 100644 index 000000000000..669712c2dea1 --- /dev/null +++ b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/model/PresignedFileDownload.java @@ -0,0 +1,33 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.transfer.s3.model; + +import java.util.concurrent.CompletableFuture; +import software.amazon.awssdk.annotations.SdkPublicApi; +import software.amazon.awssdk.annotations.ThreadSafe; + +/** + * A download transfer of a single object from S3 using a presigned URL. + * + *

Unlike {@link FileDownload}, this type does not support pause/resume. + */ +@SdkPublicApi +@ThreadSafe +public interface PresignedFileDownload extends ObjectTransfer { + + @Override + CompletableFuture completionFuture(); +} diff --git a/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/DefaultPresignedFileDownloadTest.java b/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/DefaultPresignedFileDownloadTest.java new file mode 100644 index 000000000000..b98a4cb35454 --- /dev/null +++ b/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/DefaultPresignedFileDownloadTest.java @@ -0,0 +1,30 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.transfer.s3.internal; + +import nl.jqno.equalsverifier.EqualsVerifier; +import org.junit.jupiter.api.Test; +import software.amazon.awssdk.transfer.s3.internal.model.DefaultPresignedFileDownload; + +public class DefaultPresignedFileDownloadTest { + + @Test + public void equals_hashcode() { + EqualsVerifier.forClass(DefaultPresignedFileDownload.class) + .withNonnullFields("completionFuture", "progress") + .verify(); + } +} diff --git a/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/S3TransferManagerPresignedUrlDownloadTest.java b/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/S3TransferManagerPresignedUrlDownloadTest.java new file mode 100644 index 000000000000..a299e5ca41f7 --- /dev/null +++ b/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/S3TransferManagerPresignedUrlDownloadTest.java @@ -0,0 +1,179 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.transfer.s3.internal; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import java.net.URL; +import java.nio.file.Paths; +import java.util.concurrent.CompletableFuture; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import software.amazon.awssdk.core.ResponseBytes; +import software.amazon.awssdk.core.async.AsyncResponseTransformer; +import software.amazon.awssdk.services.s3.S3AsyncClient; +import software.amazon.awssdk.services.s3.model.GetObjectResponse; +import software.amazon.awssdk.services.s3.presignedurl.AsyncPresignedUrlExtension; +import software.amazon.awssdk.services.s3.presignedurl.model.PresignedUrlDownloadRequest; +import software.amazon.awssdk.transfer.s3.model.CompletedDownload; +import software.amazon.awssdk.transfer.s3.model.CompletedFileDownload; +import software.amazon.awssdk.transfer.s3.model.PresignedDownloadFileRequest; +import software.amazon.awssdk.transfer.s3.model.PresignedDownloadRequest; + +/** + * Unit tests for S3TransferManager presigned URL download functionality. + */ +class S3TransferManagerPresignedUrlDownloadTest { + private static final String PRESIGNED_URL = "https://test-bucket.s3.amazonaws.com/test-key" + + "?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=AKID" + + "&X-Amz-Date=20260101T000000Z&X-Amz-Expires=600" + + "&X-Amz-SignedHeaders=host&X-Amz-Signature=abc123"; + + private S3AsyncClient mockS3AsyncClient; + private AsyncPresignedUrlExtension mockPresignedUrlExtension; + private GenericS3TransferManager tm; + private URL presignedUrl; + + @BeforeEach + public void methodSetup() throws Exception { + mockS3AsyncClient = mock(S3AsyncClient.class); + mockPresignedUrlExtension = mock(AsyncPresignedUrlExtension.class); + presignedUrl = new URL(PRESIGNED_URL); + + when(mockS3AsyncClient.presignedUrlExtension()).thenReturn(mockPresignedUrlExtension); + + tm = new GenericS3TransferManager(mockS3AsyncClient, + mock(UploadDirectoryHelper.class), + mock(TransferManagerConfiguration.class), + mock(DownloadDirectoryHelper.class)); + } + + @AfterEach + public void methodTeardown() { + tm.close(); + } + + @Test + void downloadFileWithPresignedUrl_withValidRequest_returnsResponse() { + GetObjectResponse response = GetObjectResponse.builder().build(); + stubGetObject(CompletableFuture.completedFuture(response)); + + CompletedFileDownload completed = tm.downloadFileWithPresignedUrl(fileDownloadRequest()) + .completionFuture().join(); + + assertThat(completed.response()).isEqualTo(response); + } + + @Test + void downloadWithPresignedUrl_withValidRequest_returnsResponse() { + ResponseBytes responseBytes = ResponseBytes.fromByteArray( + GetObjectResponse.builder().build(), "test".getBytes()); + stubGetObject(CompletableFuture.completedFuture(responseBytes)); + + CompletedDownload> completed = + tm.downloadWithPresignedUrl(bytesDownloadRequest()).completionFuture().join(); + + assertThat(completed.result()).isEqualTo(responseBytes); + } + + @Test + void downloadFileWithPresignedUrl_withConsumerBuilder_returnsResponse() { + GetObjectResponse response = GetObjectResponse.builder().build(); + stubGetObject(CompletableFuture.completedFuture(response)); + + CompletedFileDownload completed = tm.downloadFileWithPresignedUrl( + request -> request.presignedUrlDownloadRequest(p -> p.presignedUrl(presignedUrl)) + .destination(Paths.get("/tmp/test"))) + .completionFuture().join(); + + assertThat(completed.response()).isEqualTo(response); + } + + @Test + void downloadFileWithPresignedUrl_whenCancelled_shouldForwardCancellation() { + CompletableFuture s3Future = new CompletableFuture<>(); + stubGetObject(s3Future); + + CompletableFuture future = + tm.downloadFileWithPresignedUrl(fileDownloadRequest()).completionFuture(); + + future.cancel(true); + assertThat(s3Future).isCancelled(); + } + + @Test + void downloadFileWithPresignedUrl_whenRequestFails_shouldCompleteExceptionally() { + CompletableFuture failedFuture = new CompletableFuture<>(); + failedFuture.completeExceptionally(new RuntimeException("download failed")); + stubGetObject(failedFuture); + + assertThatThrownBy(() -> tm.downloadFileWithPresignedUrl(fileDownloadRequest()).completionFuture().join()) + .hasCauseInstanceOf(RuntimeException.class) + .hasMessageContaining("download failed"); + } + + @Test + void downloadWithPresignedUrl_whenCancelled_shouldForwardCancellation() { + CompletableFuture s3Future = new CompletableFuture<>(); + stubGetObject(s3Future); + + CompletableFuture>> future = + tm.downloadWithPresignedUrl(bytesDownloadRequest()).completionFuture(); + + future.cancel(true); + assertThat(s3Future).isCancelled(); + } + + @Test + void downloadFileWithPresignedUrl_withNullRequest_shouldThrowNullPointerException() { + assertThatThrownBy(() -> tm.downloadFileWithPresignedUrl((PresignedDownloadFileRequest) null)) + .isInstanceOf(NullPointerException.class); + } + + @Test + void downloadWithPresignedUrl_withNullRequest_shouldThrowNullPointerException() { + assertThatThrownBy(() -> tm.downloadWithPresignedUrl(null)) + .isInstanceOf(NullPointerException.class); + } + + private PresignedDownloadFileRequest fileDownloadRequest() { + return PresignedDownloadFileRequest.builder() + .presignedUrlDownloadRequest(PresignedUrlDownloadRequest.builder() + .presignedUrl(presignedUrl) + .build()) + .destination(Paths.get("/tmp/test")) + .build(); + } + + private PresignedDownloadRequest> bytesDownloadRequest() { + return PresignedDownloadRequest.builder() + .presignedUrlDownloadRequest(PresignedUrlDownloadRequest.builder() + .presignedUrl(presignedUrl) + .build()) + .responseTransformer(AsyncResponseTransformer.toBytes()) + .build(); + } + + private void stubGetObject(CompletableFuture future) { + when(mockPresignedUrlExtension.getObject(any(PresignedUrlDownloadRequest.class), any(AsyncResponseTransformer.class))) + .thenReturn(future); + } +} diff --git a/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/S3TransferManagerPresignedUrlListenerWiremockTest.java b/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/S3TransferManagerPresignedUrlListenerWiremockTest.java new file mode 100644 index 000000000000..2e9e5dbecca8 --- /dev/null +++ b/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/S3TransferManagerPresignedUrlListenerWiremockTest.java @@ -0,0 +1,253 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.transfer.s3.internal; + +import static com.github.tomakehurst.wiremock.client.WireMock.aResponse; +import static com.github.tomakehurst.wiremock.client.WireMock.get; +import static com.github.tomakehurst.wiremock.client.WireMock.stubFor; +import static com.github.tomakehurst.wiremock.client.WireMock.urlPathEqualTo; +import static org.assertj.core.api.Assertions.assertThatExceptionOfType; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.timeout; +import static org.mockito.Mockito.times; + +import com.github.tomakehurst.wiremock.client.WireMock; +import com.github.tomakehurst.wiremock.junit5.WireMockRuntimeInfo; +import com.github.tomakehurst.wiremock.junit5.WireMockTest; +import java.io.IOException; +import java.net.URI; +import java.net.URL; +import java.util.concurrent.CompletionException; +import java.util.stream.Stream; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import org.mockito.ArgumentMatchers; +import org.mockito.Mockito; +import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; +import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; +import software.amazon.awssdk.core.async.AsyncResponseTransformer; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.s3.S3AsyncClient; +import software.amazon.awssdk.services.s3.presignedurl.model.PresignedUrlDownloadRequest; +import software.amazon.awssdk.testutils.RandomTempFile; +import software.amazon.awssdk.transfer.s3.S3TransferManager; +import software.amazon.awssdk.transfer.s3.model.Download; +import software.amazon.awssdk.transfer.s3.model.PresignedFileDownload; +import software.amazon.awssdk.transfer.s3.model.PresignedDownloadFileRequest; +import software.amazon.awssdk.transfer.s3.model.PresignedDownloadRequest; +import software.amazon.awssdk.transfer.s3.progress.TransferListener; + +/** + * Tests that TransferListener callbacks fire correctly for presigned URL downloads + * with both multipart-enabled and non-multipart clients. + */ +@WireMockTest +public class S3TransferManagerPresignedUrlListenerWiremockTest { + + private static URI testEndpoint; + private static RandomTempFile testFile; + + @BeforeAll + public static void init(WireMockRuntimeInfo wm) throws IOException { + testEndpoint = URI.create(wm.getHttpBaseUrl()); + testFile = new RandomTempFile("presigned-listener-test", 1024); + } + + @BeforeEach + void resetWireMock() { + WireMock.reset(); + } + + private static S3AsyncClient s3AsyncClient(boolean multipartEnabled) { + return S3AsyncClient.builder() + .multipartEnabled(multipartEnabled) + .region(Region.US_EAST_1) + .endpointOverride(testEndpoint) + .credentialsProvider( + StaticCredentialsProvider.create(AwsBasicCredentials.create("key", "secret"))) + .build(); + } + + static Stream presignedUrlTestCases() { + return Stream.of( + Arguments.of(true, "toFile", null), + Arguments.of(true, "toFile", "bytes=0-511"), + Arguments.of(true, "toBytes", null), + Arguments.of(true, "toBytes", "bytes=0-511"), + Arguments.of(false, "toFile", null), + Arguments.of(false, "toFile", "bytes=0-511"), + Arguments.of(false, "toBytes", null), + Arguments.of(false, "toBytes", "bytes=0-511") + ); + } + + @ParameterizedTest(name = "presignedUrlDownload_multipart={0}_type={1}_range={2}") + @MethodSource("presignedUrlTestCases") + void presignedUrlDownload_shouldInvokeListener(boolean multipartEnabled, String type, String range) throws Exception { + S3AsyncClient s3Async = s3AsyncClient(multipartEnabled); + S3TransferManager tm = new GenericS3TransferManager(s3Async, mock(UploadDirectoryHelper.class), + mock(TransferManagerConfiguration.class), + mock(DownloadDirectoryHelper.class)); + + byte[] responseBody = new byte[512]; + stubFor(get(urlPathEqualTo("/presigned-key")).willReturn(aResponse() + .withStatus(206) + .withHeader("Content-Length", "512") + .withHeader("Content-Range", "bytes 0-511/512") + .withHeader("ETag", "\"test-etag\"") + .withBody(responseBody))); + + TransferListener listener = mock(TransferListener.class); + URL presignedUrl = new URL(testEndpoint + "/presigned-key?X-Amz-Algorithm=AWS4-HMAC-SHA256"); + + PresignedUrlDownloadRequest.Builder requestBuilder = PresignedUrlDownloadRequest.builder() + .presignedUrl(presignedUrl); + if (range != null) { + requestBuilder.range(range); + } + + if ("toFile".equals(type)) { + PresignedFileDownload download = tm.downloadFileWithPresignedUrl( + PresignedDownloadFileRequest.builder() + .presignedUrlDownloadRequest(requestBuilder.build()) + .destination(testFile.toPath()) + .addTransferListener(listener) + .build()); + download.completionFuture().join(); + } else { + Download download = tm.downloadWithPresignedUrl( + PresignedDownloadRequest.builder() + .presignedUrlDownloadRequest(requestBuilder.build()) + .responseTransformer(AsyncResponseTransformer.toBytes()) + .addTransferListener(listener) + .build()); + download.completionFuture().join(); + } + + Mockito.verify(listener, timeout(1000).times(1)).transferInitiated(ArgumentMatchers.any()); + Mockito.verify(listener, timeout(1000).atLeastOnce()).bytesTransferred(ArgumentMatchers.any()); + + tm.close(); + s3Async.close(); + } + + static Stream presignedUrlFailureTestCases() { + return Stream.of( + Arguments.of(true, "toFile"), + Arguments.of(true, "toBytes"), + Arguments.of(false, "toFile"), + Arguments.of(false, "toBytes") + ); + } + + @ParameterizedTest(name = "presignedUrlDownload_failure_multipart={0}_type={1}") + @MethodSource("presignedUrlFailureTestCases") + void presignedUrlDownload_failure_shouldInvokeListener(boolean multipartEnabled, String type) throws Exception { + S3AsyncClient s3Async = s3AsyncClient(multipartEnabled); + S3TransferManager tm = new GenericS3TransferManager(s3Async, mock(UploadDirectoryHelper.class), + mock(TransferManagerConfiguration.class), + mock(DownloadDirectoryHelper.class)); + + stubFor(get(urlPathEqualTo("/presigned-key")) + .willReturn(aResponse().withStatus(404) + .withBody("TestErrorTest failure"))); + + TransferListener listener = mock(TransferListener.class); + URL presignedUrl = new URL(testEndpoint + "/presigned-key?X-Amz-Algorithm=AWS4-HMAC-SHA256"); + + if ("toFile".equals(type)) { + PresignedFileDownload download = tm.downloadFileWithPresignedUrl( + PresignedDownloadFileRequest.builder() + .presignedUrlDownloadRequest(PresignedUrlDownloadRequest.builder() + .presignedUrl(presignedUrl) + .build()) + .destination(testFile.toPath()) + .addTransferListener(listener) + .build()); + assertThatExceptionOfType(CompletionException.class).isThrownBy(() -> download.completionFuture().join()); + } else { + Download download = tm.downloadWithPresignedUrl( + PresignedDownloadRequest.builder() + .presignedUrlDownloadRequest(PresignedUrlDownloadRequest.builder() + .presignedUrl(presignedUrl) + .build()) + .responseTransformer(AsyncResponseTransformer.toBytes()) + .addTransferListener(listener) + .build()); + assertThatExceptionOfType(CompletionException.class).isThrownBy(() -> download.completionFuture().join()); + } + + Mockito.verify(listener, timeout(1000).times(1)).transferInitiated(ArgumentMatchers.any()); + Mockito.verify(listener, timeout(1000).times(1)).transferFailed(ArgumentMatchers.any()); + Mockito.verify(listener, times(0)).transferComplete(ArgumentMatchers.any()); + + tm.close(); + s3Async.close(); + } + + @ParameterizedTest(name = "presignedUrlDownload_cancelled_multipart={0}_type={1}") + @MethodSource("presignedUrlFailureTestCases") + void presignedUrlDownload_cancelled_shouldInvokeTransferFailed(boolean multipartEnabled, String type) throws Exception { + S3AsyncClient s3Async = s3AsyncClient(multipartEnabled); + S3TransferManager tm = new GenericS3TransferManager(s3Async, mock(UploadDirectoryHelper.class), + mock(TransferManagerConfiguration.class), + mock(DownloadDirectoryHelper.class)); + + // Slow response to keep request in-flight during cancellation + stubFor(get(urlPathEqualTo("/presigned-key")).willReturn(aResponse() + .withStatus(206) + .withHeader("Content-Length", "512") + .withHeader("Content-Range", "bytes 0-511/512") + .withHeader("ETag", "\"test-etag\"") + .withBody(new byte[512]) + .withFixedDelay(5000))); + + TransferListener listener = mock(TransferListener.class); + URL presignedUrl = new URL(testEndpoint + "/presigned-key?X-Amz-Algorithm=AWS4-HMAC-SHA256"); + + if ("toFile".equals(type)) { + PresignedFileDownload download = tm.downloadFileWithPresignedUrl( + PresignedDownloadFileRequest.builder() + .presignedUrlDownloadRequest(PresignedUrlDownloadRequest.builder() + .presignedUrl(presignedUrl) + .build()) + .destination(testFile.toPath()) + .addTransferListener(listener) + .build()); + download.completionFuture().cancel(true); + } else { + Download download = tm.downloadWithPresignedUrl( + PresignedDownloadRequest.builder() + .presignedUrlDownloadRequest(PresignedUrlDownloadRequest.builder() + .presignedUrl(presignedUrl) + .build()) + .responseTransformer(AsyncResponseTransformer.toBytes()) + .addTransferListener(listener) + .build()); + download.completionFuture().cancel(true); + } + + Mockito.verify(listener, timeout(1000).times(1)).transferInitiated(ArgumentMatchers.any()); + Mockito.verify(listener, timeout(1000).times(1)).transferFailed(ArgumentMatchers.any()); + Mockito.verify(listener, times(0)).transferComplete(ArgumentMatchers.any()); + + tm.close(); + s3Async.close(); + } +} diff --git a/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/model/PresignedDownloadFileRequestTest.java b/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/model/PresignedDownloadFileRequestTest.java new file mode 100644 index 000000000000..fb437e5f8796 --- /dev/null +++ b/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/model/PresignedDownloadFileRequestTest.java @@ -0,0 +1,119 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.transfer.s3.model; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import java.io.File; +import java.net.URL; +import java.nio.file.Path; +import java.nio.file.Paths; +import nl.jqno.equalsverifier.EqualsVerifier; +import org.junit.jupiter.api.Test; +import software.amazon.awssdk.services.s3.presignedurl.model.PresignedUrlDownloadRequest; + +class PresignedDownloadFileRequestTest { + + @Test + void build_withoutPresignedUrlDownloadRequest_throwsNullPointerException() { + assertThatThrownBy(() -> PresignedDownloadFileRequest.builder() + .destination(Paths.get(".")) + .build()) + .isInstanceOf(NullPointerException.class) + .hasMessageContaining("presignedUrlDownloadRequest"); + } + + @Test + void build_withoutDestination_throwsNullPointerException() { + assertThatThrownBy(() -> PresignedDownloadFileRequest.builder() + .presignedUrlDownloadRequest(createPresignedRequest()) + .build()) + .isInstanceOf(NullPointerException.class) + .hasMessageContaining("destination"); + } + + @Test + void build_withPath_setsDestinationCorrectly() { + Path path = Paths.get("."); + PresignedUrlDownloadRequest presignedRequest = createPresignedRequest(); + + PresignedDownloadFileRequest request = PresignedDownloadFileRequest.builder() + .destination(path) + .presignedUrlDownloadRequest(presignedRequest) + .build(); + + assertThat(request.destination()).isEqualTo(path); + assertThat(request.presignedUrlDownloadRequest()).isEqualTo(presignedRequest); + } + + @Test + void build_withFile_convertsToPathCorrectly() { + Path path = Paths.get("."); + PresignedUrlDownloadRequest presignedRequest = createPresignedRequest(); + + PresignedDownloadFileRequest request = PresignedDownloadFileRequest.builder() + .destination(path.toFile()) + .presignedUrlDownloadRequest(presignedRequest) + .build(); + + assertThat(request.destination()).isEqualTo(path); + } + + @Test + void destination_withNullFile_throwsNullPointerException() { + File file = null; + + assertThatThrownBy(() -> PresignedDownloadFileRequest.builder() + .destination(file)) + .isInstanceOf(NullPointerException.class) + .hasMessageContaining("destination"); + } + + @Test + void presignedUrlDownloadRequest_withConsumerBuilder_buildsCorrectly() { + Path path = Paths.get("."); + + PresignedDownloadFileRequest request = PresignedDownloadFileRequest.builder() + .destination(path) + .presignedUrlDownloadRequest(b -> b.presignedUrl(createTestUrl())) + .build(); + + assertThat(request.presignedUrlDownloadRequest()).isNotNull(); + assertThat(request.presignedUrlDownloadRequest().presignedUrl()).isEqualTo(createTestUrl()); + } + + @Test + void equalsHashCodeTest() { + EqualsVerifier.forClass(PresignedDownloadFileRequest.class) + .withNonnullFields("destination", "presignedUrlDownloadRequest") + .verify(); + } + + private PresignedUrlDownloadRequest createPresignedRequest() { + return PresignedUrlDownloadRequest.builder() + .presignedUrl(createTestUrl()) + .build(); + } + + private URL createTestUrl() { + try { + return new URL("https://example.com/test"); + } catch (Exception e) { + throw new RuntimeException(e); + } + } +} \ No newline at end of file diff --git a/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/model/PresignedDownloadRequestTest.java b/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/model/PresignedDownloadRequestTest.java new file mode 100644 index 000000000000..15381bfd50b6 --- /dev/null +++ b/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/model/PresignedDownloadRequestTest.java @@ -0,0 +1,136 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.transfer.s3.model; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import java.net.URL; +import nl.jqno.equalsverifier.EqualsVerifier; +import org.junit.jupiter.api.Test; +import software.amazon.awssdk.core.ResponseBytes; +import software.amazon.awssdk.core.async.AsyncResponseTransformer; +import software.amazon.awssdk.services.s3.model.GetObjectResponse; +import software.amazon.awssdk.services.s3.presignedurl.model.PresignedUrlDownloadRequest; + +class PresignedDownloadRequestTest { + + @Test + void build_withoutPresignedUrlDownloadRequest_throwsNullPointerException() { + assertThatThrownBy(() -> PresignedDownloadRequest.builder() + .responseTransformer(AsyncResponseTransformer.toBytes()) + .build()) + .isInstanceOf(NullPointerException.class) + .hasMessageContaining("presignedUrlDownloadRequest"); + } + + @Test + void builder_withoutResponseTransformer_returnsUntypedBuilder() { + PresignedDownloadRequest.UntypedBuilder untypedBuilder = PresignedDownloadRequest.builder() + .presignedUrlDownloadRequest(createPresignedRequest()); + + assertThat(untypedBuilder).isNotNull(); + } + + @Test + void build_withResponseTransformer_createsRequestCorrectly() { + AsyncResponseTransformer> responseTransformer = + AsyncResponseTransformer.toBytes(); + PresignedUrlDownloadRequest presignedRequest = createPresignedRequest(); + + PresignedDownloadRequest> request = + PresignedDownloadRequest.builder() + .presignedUrlDownloadRequest(presignedRequest) + .responseTransformer(responseTransformer) + .build(); + + assertThat(request.responseTransformer()).isEqualTo(responseTransformer); + assertThat(request.presignedUrlDownloadRequest()).isEqualTo(presignedRequest); + } + + @Test + void presignedUrlDownloadRequest_withConsumerBuilder_buildsCorrectly() { + AsyncResponseTransformer> responseTransformer = + AsyncResponseTransformer.toBytes(); + + PresignedDownloadRequest> request = + PresignedDownloadRequest.builder() + .presignedUrlDownloadRequest(b -> b.presignedUrl(createTestUrl())) + .responseTransformer(responseTransformer) + .build(); + + assertThat(request.presignedUrlDownloadRequest()).isNotNull(); + assertThat(request.presignedUrlDownloadRequest().presignedUrl()).isEqualTo(createTestUrl()); + } + + @Test + void responseTransformer_withNull_throwsNullPointerException() { + assertThatThrownBy(() -> PresignedDownloadRequest.builder() + .presignedUrlDownloadRequest(createPresignedRequest()) + .responseTransformer(null) + .build()) + .isInstanceOf(NullPointerException.class) + .hasMessageContaining("responseTransformer"); + } + + @Test + void builder_untypedToTypedTransition_worksCorrectly() { + PresignedDownloadRequest.UntypedBuilder untypedBuilder = PresignedDownloadRequest.builder() + .presignedUrlDownloadRequest(createPresignedRequest()); + + PresignedDownloadRequest> request = + untypedBuilder.responseTransformer(AsyncResponseTransformer.toBytes()) + .build(); + + assertThat(request.responseTransformer()).isNotNull(); + assertThat(request.presignedUrlDownloadRequest()).isNotNull(); + } + + @Test + void toBuilder_copiesAllFields() { + PresignedDownloadRequest> original = + PresignedDownloadRequest.builder() + .presignedUrlDownloadRequest(createPresignedRequest()) + .responseTransformer(AsyncResponseTransformer.toBytes()) + .build(); + + PresignedDownloadRequest> copy = original.toBuilder().build(); + + assertThat(copy.responseTransformer()).isEqualTo(original.responseTransformer()); + assertThat(copy.presignedUrlDownloadRequest()).isEqualTo(original.presignedUrlDownloadRequest()); + } + + @Test + void equalsHashCodeTest() { + EqualsVerifier.forClass(PresignedDownloadRequest.class) + .withNonnullFields("responseTransformer", "presignedUrlDownloadRequest") + .verify(); + } + + private PresignedUrlDownloadRequest createPresignedRequest() { + return PresignedUrlDownloadRequest.builder() + .presignedUrl(createTestUrl()) + .build(); + } + + private URL createTestUrl() { + try { + return new URL("https://example.com/test"); + } catch (Exception e) { + throw new RuntimeException(e); + } + } +} \ No newline at end of file diff --git a/services/s3/src/it/java/software/amazon/awssdk/services/s3/presignedurl/AsyncPresignedUrlExtensionIntegrationTest.java b/services/s3/src/it/java/software/amazon/awssdk/services/s3/presignedurl/AsyncPresignedUrlExtensionIntegrationTest.java new file mode 100644 index 000000000000..1ec2cf2df702 --- /dev/null +++ b/services/s3/src/it/java/software/amazon/awssdk/services/s3/presignedurl/AsyncPresignedUrlExtensionIntegrationTest.java @@ -0,0 +1,32 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ +package software.amazon.awssdk.services.s3.presignedurl; + +import org.junit.jupiter.api.BeforeAll; +import software.amazon.awssdk.services.s3.S3AsyncClient; + +public class AsyncPresignedUrlExtensionIntegrationTest extends AsyncPresignedUrlExtensionTestSuite { + + @BeforeAll + static void setUpIntegrationTest() { + S3AsyncClient s3AsyncClient = s3AsyncClientBuilder().build(); + presignedUrlExtension = s3AsyncClient.presignedUrlExtension(); + } + + @Override + protected S3AsyncClient createS3AsyncClient() { + return s3AsyncClientBuilder().build(); + } +} diff --git a/services/s3/src/it/java/software/amazon/awssdk/services/s3/presignedurl/AsyncPresignedUrlExtensionMultipartIntegrationTest.java b/services/s3/src/it/java/software/amazon/awssdk/services/s3/presignedurl/AsyncPresignedUrlExtensionMultipartIntegrationTest.java new file mode 100644 index 000000000000..16e41325817f --- /dev/null +++ b/services/s3/src/it/java/software/amazon/awssdk/services/s3/presignedurl/AsyncPresignedUrlExtensionMultipartIntegrationTest.java @@ -0,0 +1,35 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ +package software.amazon.awssdk.services.s3.presignedurl; + +import org.junit.jupiter.api.BeforeAll; +import software.amazon.awssdk.services.s3.S3AsyncClient; + + +public class AsyncPresignedUrlExtensionMultipartIntegrationTest extends AsyncPresignedUrlExtensionTestSuite { + + @BeforeAll + static void setUpIntegrationTest() { + S3AsyncClient s3AsyncClient = s3AsyncClientBuilder() + .multipartEnabled(true) + .build(); + presignedUrlExtension = s3AsyncClient.presignedUrlExtension(); + } + + @Override + protected S3AsyncClient createS3AsyncClient() { + return s3AsyncClientBuilder().build(); + } +} diff --git a/services/s3/src/it/java/software/amazon/awssdk/services/s3/presignedurl/AsyncPresignedUrlExtensionTestSuite.java b/services/s3/src/it/java/software/amazon/awssdk/services/s3/presignedurl/AsyncPresignedUrlExtensionTestSuite.java new file mode 100644 index 000000000000..b5a3edbfd8f7 --- /dev/null +++ b/services/s3/src/it/java/software/amazon/awssdk/services/s3/presignedurl/AsyncPresignedUrlExtensionTestSuite.java @@ -0,0 +1,534 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ +package software.amazon.awssdk.services.s3.presignedurl; + +import static org.apache.commons.lang3.RandomStringUtils.randomAscii; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.AssertionsForClassTypes.assertThatThrownBy; + +import java.io.ByteArrayInputStream; +import java.net.URL; +import java.nio.charset.StandardCharsets; +import java.nio.file.Path; +import java.time.Duration; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Random; +import java.util.UUID; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.TimeUnit; +import java.util.stream.Stream; + +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +import software.amazon.awssdk.core.ResponseBytes; +import software.amazon.awssdk.core.async.AsyncRequestBody; +import software.amazon.awssdk.core.async.AsyncResponseTransformer; +import software.amazon.awssdk.core.sync.RequestBody; +import software.amazon.awssdk.metrics.MetricCollection; +import software.amazon.awssdk.metrics.MetricPublisher; +import software.amazon.awssdk.services.s3.S3AsyncClient; +import software.amazon.awssdk.services.s3.S3IntegrationTestBase; +import software.amazon.awssdk.services.s3.model.ChecksumAlgorithm; +import software.amazon.awssdk.services.s3.model.ChecksumMode; +import software.amazon.awssdk.services.s3.model.CompletedMultipartUpload; +import software.amazon.awssdk.services.s3.model.CompletedPart; +import software.amazon.awssdk.services.s3.model.CreateMultipartUploadResponse; +import software.amazon.awssdk.services.s3.model.DeleteObjectRequest; +import software.amazon.awssdk.services.s3.model.GetObjectResponse; +import software.amazon.awssdk.services.s3.model.PutObjectRequest; +import software.amazon.awssdk.services.s3.model.S3Exception; +import software.amazon.awssdk.services.s3.model.UploadPartResponse; +import software.amazon.awssdk.services.s3.presigner.S3Presigner; +import software.amazon.awssdk.services.s3.presigner.model.PresignedGetObjectRequest; +import software.amazon.awssdk.services.s3.presignedurl.model.PresignedUrlDownloadRequest; +import software.amazon.awssdk.services.s3.utils.S3TestUtils; +import software.amazon.awssdk.testutils.service.S3BucketUtils; +import software.amazon.awssdk.utils.Md5Utils; + +/** + * Abstract test suite for AsyncPresignedUrlExtension integration tests. + */ +public abstract class AsyncPresignedUrlExtensionTestSuite extends S3IntegrationTestBase { + protected static S3Presigner presigner; + protected static AsyncPresignedUrlExtension presignedUrlExtension; + protected static String testBucket; + + @TempDir + static Path temporaryFolder; + + protected static String testGetObjectKey; + protected static String testLargeObjectKey; + protected static String testMpuChecksumKey; + protected static String testObjectContent; + protected static byte[] testLargeObjectContent; + protected static byte[] testMpuObjectContent; + protected static String expectedLargeObjectMd5; + + protected abstract S3AsyncClient createS3AsyncClient(); + + @BeforeAll + static void setUpTestSuite() throws Exception { + setUp(); + + presigner = S3Presigner.builder() + .region(DEFAULT_REGION) + .credentialsProvider(CREDENTIALS_PROVIDER_CHAIN) + .build(); + testBucket = S3BucketUtils.temporaryBucketName("async-presigned-url-extension-test"); + createBucket(testBucket); + testGetObjectKey = generateRandomObjectKey(); + testLargeObjectKey = generateRandomObjectKey() + "-large"; + testObjectContent = "Hello AsyncPresignedUrlExtension Integration Test"; + testLargeObjectContent = randomAscii(5 * 1024 * 1024).getBytes(StandardCharsets.UTF_8); + + try (ByteArrayInputStream originalStream = new ByteArrayInputStream(testLargeObjectContent)) { + expectedLargeObjectMd5 = Md5Utils.md5AsBase64(originalStream); + } catch (Exception e) { + throw new RuntimeException("Failed to compute MD5 for test data", e); + } + + S3TestUtils.putObject(AsyncPresignedUrlExtensionTestSuite.class, s3, testBucket, testGetObjectKey, testObjectContent); + s3Async.putObject( + PutObjectRequest.builder() + .bucket(testBucket) + .key(testLargeObjectKey) + .build(), + AsyncRequestBody.fromBytes(testLargeObjectContent) + ).join(); + uploadMpuObjectWithChecksum(); + S3TestUtils.addCleanupTask(AsyncPresignedUrlExtensionTestSuite.class, () -> { + s3.deleteObject(DeleteObjectRequest.builder() + .bucket(testBucket) + .key(testGetObjectKey) + .build()); + s3.deleteObject(DeleteObjectRequest.builder() + .bucket(testBucket) + .key(testLargeObjectKey) + .build()); + s3.deleteObject(DeleteObjectRequest.builder() + .bucket(testBucket) + .key(testMpuChecksumKey) + .build()); + deleteBucketAndAllContents(testBucket); + }); + } + + @AfterAll + static void tearDownTestSuite() { + try { + S3TestUtils.runCleanupTasks(AsyncPresignedUrlExtensionTestSuite.class); + } catch (Exception e) { + } + if (presigner != null) { + presigner.close(); + } + cleanUpResources(); + } + + @ParameterizedTest(name = "{0}") + @MethodSource("basicFunctionalityTestData") + void getObject_withValidPresignedUrl_returnsContent(String testDescription, + String objectKey, + String expectedContent) throws Exception { + PresignedUrlDownloadRequest request = createRequestForKey(objectKey); + + CompletableFuture> future = + presignedUrlExtension.getObject(request, AsyncResponseTransformer.toBytes()); + ResponseBytes response = future.get(); + + assertThat(response).isNotNull(); + if (expectedContent != null) { + assertThat(response.asUtf8String()).isEqualTo(expectedContent); + assertThat(response.response().contentLength()).isEqualTo(expectedContent.length()); + } else { + try (ByteArrayInputStream downloadedStream = new ByteArrayInputStream(response.asByteArray())) { + String downloadedMd5 = Md5Utils.md5AsBase64(downloadedStream); + assertThat(downloadedMd5).isEqualTo(expectedLargeObjectMd5); + assertThat(response.asByteArray().length).isEqualTo(testLargeObjectContent.length); + } + assertThat(response.response()).isNotNull(); + assertThat(response.response().contentLength()).isEqualTo((long) testLargeObjectContent.length); + assertThat(response.response().sdkHttpResponse().firstMatchingHeader("x-amz-request-id")).isPresent(); + } + } + + @Test + void getObject_withValidPresignedUrl_savesContentToFile() throws Exception { + PresignedUrlDownloadRequest request = createRequestForKey(testGetObjectKey); + Path downloadFile = temporaryFolder.resolve("download-" + UUID.randomUUID() + ".txt"); + CompletableFuture future = + presignedUrlExtension.getObject(request, downloadFile); + GetObjectResponse response = future.get(); + + assertThat(response).isNotNull(); + assertThat(downloadFile).exists(); + assertThat(downloadFile).hasContent(testObjectContent); + } + + @ParameterizedTest(name = "{0}") + @MethodSource("rangeTestData") + void getObject_withRangeRequest_returnsSpecifiedRange(String testDescription, + String range, + String expectedContent) throws Exception { + PresignedUrlDownloadRequest request = createRequestForKey(testGetObjectKey, range); + + CompletableFuture> future = + presignedUrlExtension.getObject(request, AsyncResponseTransformer.toBytes()); + ResponseBytes response = future.get(); + + assertThat(response.asUtf8String()).isEqualTo(expectedContent); + } + + @Test + void getObject_withMultipleRangeRequestsConcurrently_returnsCorrectContent() throws Exception { + String concurrentTestKey = uploadTestObject("concurrent-test", "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ"); + List>> futures = new ArrayList<>(); + + futures.add(presignedUrlExtension.getObject( + createRequestForKey(concurrentTestKey, "bytes=0-8"), // "012345678" + AsyncResponseTransformer.toBytes())); + futures.add(presignedUrlExtension.getObject( + createRequestForKey(concurrentTestKey, "bytes=9-17"), // "9ABCDEFGH" + AsyncResponseTransformer.toBytes())); + futures.add(presignedUrlExtension.getObject( + createRequestForKey(concurrentTestKey, "bytes=18-26"), // "IJKLMNOPQ" + AsyncResponseTransformer.toBytes())); + futures.add(presignedUrlExtension.getObject( + createRequestForKey(concurrentTestKey, "bytes=27-35"), // "RSTUVWXYZ" + AsyncResponseTransformer.toBytes())); + + CompletableFuture allFutures = CompletableFuture.allOf( + futures.toArray(new CompletableFuture[0])); + allFutures.get(30, TimeUnit.SECONDS); + + StringBuilder result = new StringBuilder(); + for (CompletableFuture> future : futures) { + result.append(future.get().asUtf8String()); + } + + assertThat(result.toString()).isEqualTo("0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ"); + } + + @Test + void getObject_withLargeObjectToFile_savesCompleteContentAndCollectsMetrics() throws Exception { + List collectedMetrics = new ArrayList<>(); + MetricPublisher metricPublisher = new MetricPublisher() { + @Override + public void publish(MetricCollection metricCollection) { + collectedMetrics.add(metricCollection); + } + @Override + public void close() {} + }; + + try (S3AsyncClient clientWithMetrics = s3AsyncClientBuilder() + .overrideConfiguration(o -> o.addMetricPublisher(metricPublisher)) + .build()) { + + AsyncPresignedUrlExtension metricsExtension = clientWithMetrics.presignedUrlExtension(); + PresignedUrlDownloadRequest request = createRequestForKey(testLargeObjectKey); + Path downloadFile = temporaryFolder.resolve("large-download-with-metrics-" + UUID.randomUUID() + ".bin"); + + CompletableFuture future = + metricsExtension.getObject(request, downloadFile); + GetObjectResponse response = future.get(60, TimeUnit.SECONDS); + + assertThat(response).isNotNull(); + assertThat(downloadFile).exists(); + assertThat(downloadFile.toFile().length()).isEqualTo(testLargeObjectContent.length); + assertThat(response.contentLength()).isEqualTo((long) testLargeObjectContent.length); + assertThat(response.sdkHttpResponse().firstMatchingHeader("x-amz-request-id")).isPresent(); + assertThat(collectedMetrics).isNotEmpty(); + } + } + + @Test + void getObject_emptyObject_toBytes_shouldSucceed() throws Exception { + String emptyKey = uploadTestObject("empty-object-bytes", ""); + + PresignedUrlDownloadRequest request = createRequestForKey(emptyKey); + ResponseBytes response = + presignedUrlExtension.getObject(request, AsyncResponseTransformer.toBytes()) + .get(30, TimeUnit.SECONDS); + + assertThat(response.asByteArray()).isEmpty(); + assertThat(response.response().contentLength()).isEqualTo(0L); + } + + @Test + void getObject_emptyObject_toFile_shouldSucceed() throws Exception { + String emptyKey = uploadTestObject("empty-object-file", ""); + + PresignedUrlDownloadRequest request = createRequestForKey(emptyKey); + Path downloadFile = temporaryFolder.resolve("empty-download-" + UUID.randomUUID() + ".bin"); + GetObjectResponse response = + presignedUrlExtension.getObject(request, downloadFile) + .get(30, TimeUnit.SECONDS); + + assertThat(downloadFile).exists(); + assertThat(downloadFile.toFile().length()).isEqualTo(0L); + } + + @Test + void getObject_emptyObject_withRange_shouldThrow416() throws Exception { + String emptyKey = uploadTestObject("empty-object-range", ""); + + PresignedUrlDownloadRequest request = createRequestForKey(emptyKey, "bytes=0-1024"); + + assertThatThrownBy(() -> presignedUrlExtension.getObject(request, AsyncResponseTransformer.toBytes()) + .get(30, TimeUnit.SECONDS)) + .hasCauseInstanceOf(S3Exception.class); + } + + @Test + void getObject_withChecksumModeEnabled_returnsChecksumHeaders() throws Exception { + PresignedGetObjectRequest presigned = presigner.presignGetObject(r -> r + .getObjectRequest(req -> req.bucket(testBucket).key(testGetObjectKey) + .checksumMode(ChecksumMode.ENABLED)) + .signatureDuration(Duration.ofMinutes(10))); + + PresignedUrlDownloadRequest request = PresignedUrlDownloadRequest.builder() + .presignedUrl(presigned.url()) + .build(); + + ResponseBytes response = + presignedUrlExtension.getObject(request, AsyncResponseTransformer.toBytes()) + .get(30, TimeUnit.SECONDS); + + assertThat(response.asUtf8String()).isEqualTo(testObjectContent); + assertThat(response.response().checksumTypeAsString()).isNotNull(); + } + + @Test + void getObject_withoutChecksumMode_doesNotReturnChecksumHeaders() throws Exception { + PresignedUrlDownloadRequest request = createRequestForKey(testGetObjectKey); + + ResponseBytes response = + presignedUrlExtension.getObject(request, AsyncResponseTransformer.toBytes()) + .get(30, TimeUnit.SECONDS); + + assertThat(response.asUtf8String()).isEqualTo(testObjectContent); + assertThat(response.response().checksumTypeAsString()).isNull(); + } + + @Test + void getObject_withChecksumModeEnabled_requestContainsChecksumHeader() throws Exception { + PresignedGetObjectRequest presigned = presigner.presignGetObject(r -> r + .getObjectRequest(req -> req.bucket(testBucket).key(testGetObjectKey) + .checksumMode(ChecksumMode.ENABLED)) + .signatureDuration(Duration.ofMinutes(10))); + + // Verify the presigned URL has checksum-mode in SignedHeaders + assertThat(presigned.signedHeaders()).containsKey("x-amz-checksum-mode"); + assertThat(presigned.isBrowserExecutable()).isFalse(); + + PresignedUrlDownloadRequest request = PresignedUrlDownloadRequest.builder() + .presignedUrl(presigned.url()) + .build(); + + // Download should succeed (proves marshaller sent the header) + ResponseBytes response = + presignedUrlExtension.getObject(request, AsyncResponseTransformer.toBytes()) + .get(30, TimeUnit.SECONDS); + + assertThat(response).isNotNull(); + assertThat(response.asUtf8String()).isEqualTo(testObjectContent); + } + + static Stream basicFunctionalityTestData() { + return Stream.of( + Arguments.of("getObject_withValidUrl_returnsContent", + testGetObjectKey, testObjectContent), + Arguments.of("getObject_withValidLargeObjectUrl_returnsContent", + testLargeObjectKey, null) + ); + } + + static Stream rangeTestData() { + String content = "Hello AsyncPresignedUrlExtension Integration Test"; + return Stream.of( + Arguments.of("getObject_withPrefix10BytesRange_returnsFirst10Bytes", + "bytes=0-9", content.substring(0, 10)), + Arguments.of("getObject_withSuffix10BytesRange_returnsLast10Bytes", + "bytes=-10", content.substring(content.length() - 10)), + Arguments.of("getObject_withMiddle10BytesRange_returnsMiddle10Bytes", + "bytes=10-19", content.substring(10, 20)), + Arguments.of("getObject_withSingleByteRange_returnsSingleByte", + "bytes=0-0", content.substring(0, 1)) + ); + } + + @Test + void getObject_withRangeRequest_preservesPartialMetadata() throws Exception { + PresignedUrlDownloadRequest request = createRequestForKey(testLargeObjectKey, "bytes=0-1048575"); + ResponseBytes response = + presignedUrlExtension.getObject(request, AsyncResponseTransformer.toBytes()) + .get(30, TimeUnit.SECONDS); + + assertThat(response.response().contentLength()).isEqualTo(1048576L); + assertThat(response.response().contentRange()).isNotNull(); + assertThat(response.response().sdkHttpResponse().firstMatchingHeader("x-amz-request-id")).isPresent(); + } + + @ParameterizedTest(name = "getObject_largeObject_{0}_hasCorrectFullObjectMetadata") + @MethodSource("transformerTypes") + void getObject_largeObject_hasCorrectFullObjectMetadata(String type) throws Exception { + PresignedUrlDownloadRequest request = createRequestForKey(testLargeObjectKey); + + if ("toFile".equals(type)) { + Path downloadFile = temporaryFolder.resolve("large-metadata-test-" + UUID.randomUUID() + ".bin"); + GetObjectResponse response = + presignedUrlExtension.getObject(request, downloadFile) + .get(60, TimeUnit.SECONDS); + + assertThat(response.contentLength()).isEqualTo((long) testLargeObjectContent.length); + assertThat(downloadFile.toFile().length()).isEqualTo(testLargeObjectContent.length); + assertThat(response.sdkHttpResponse().firstMatchingHeader("x-amz-request-id")).isPresent(); + } else { + ResponseBytes response = + presignedUrlExtension.getObject(request, AsyncResponseTransformer.toBytes()) + .get(60, TimeUnit.SECONDS); + + assertThat(response.asByteArray().length).isEqualTo(testLargeObjectContent.length); + assertThat(response.response().contentLength()).isEqualTo((long) testLargeObjectContent.length); + assertThat(response.response().sdkHttpResponse().firstMatchingHeader("x-amz-request-id")).isPresent(); + } + } + + static Stream transformerTypes() { + return Stream.of("toFile", "toBytes"); + } + + @ParameterizedTest(name = "getObject_mpuObject_{0}_hasCorrectMetadata") + @MethodSource("checksumModes") + void getObject_mpuObject_hasCorrectMetadata(String mode) throws Exception { + PresignedUrlDownloadRequest request; + if ("withChecksumMode".equals(mode)) { + PresignedGetObjectRequest presigned = presigner.presignGetObject(r -> r + .getObjectRequest(req -> req.bucket(testBucket).key(testMpuChecksumKey) + .checksumMode(ChecksumMode.ENABLED)) + .signatureDuration(Duration.ofMinutes(10))); + request = PresignedUrlDownloadRequest.builder().presignedUrl(presigned.url()).build(); + } else { + request = PresignedUrlDownloadRequest.builder() + .presignedUrl(createPresignedUrl(testMpuChecksumKey)) + .build(); + } + + ResponseBytes response = + presignedUrlExtension.getObject(request, AsyncResponseTransformer.toBytes()) + .get(60, TimeUnit.SECONDS); + + assertThat(response.asByteArray().length).isEqualTo(testMpuObjectContent.length); + assertThat(response.response().contentLength()).isEqualTo((long) testMpuObjectContent.length); + assertThat(response.response().sdkHttpResponse().firstMatchingHeader("x-amz-request-id")).isPresent(); + } + + static Stream checksumModes() { + return Stream.of("withChecksumMode", "withoutChecksumMode"); + } + + @Test + void getObject_mpuObjectWithRange_preservesPartialMetadata() throws Exception { + PresignedUrlDownloadRequest request = PresignedUrlDownloadRequest.builder() + .presignedUrl(createPresignedUrl(testMpuChecksumKey)) + .range("bytes=0-1048575") + .build(); + + ResponseBytes response = + presignedUrlExtension.getObject(request, AsyncResponseTransformer.toBytes()) + .get(30, TimeUnit.SECONDS); + + assertThat(response.response().contentLength()).isEqualTo(1048576L); + assertThat(response.response().contentRange()).contains("bytes 0-1048575/"); + assertThat(response.response().sdkHttpResponse().firstMatchingHeader("x-amz-request-id")).isPresent(); + } + + // Helper methods + private static void uploadMpuObjectWithChecksum() { + testMpuChecksumKey = generateRandomObjectKey() + "-mpu-checksum"; + int partSize = 5 * 1024 * 1024; + int numParts = 2; + testMpuObjectContent = new byte[partSize * numParts]; + new Random(42).nextBytes(testMpuObjectContent); + + CreateMultipartUploadResponse createResp = + s3.createMultipartUpload(b -> b.bucket(testBucket).key(testMpuChecksumKey) + .checksumAlgorithm(ChecksumAlgorithm.CRC32)); + String uploadId = createResp.uploadId(); + List parts = new ArrayList<>(); + + for (int i = 0; i < numParts; i++) { + byte[] partData = Arrays.copyOfRange(testMpuObjectContent, i * partSize, (i + 1) * partSize); + final int partNum = i + 1; + UploadPartResponse uploadResp = s3.uploadPart( + b -> b.bucket(testBucket).key(testMpuChecksumKey).uploadId(uploadId).partNumber(partNum) + .checksumAlgorithm(ChecksumAlgorithm.CRC32), + RequestBody.fromBytes(partData)); + parts.add(CompletedPart.builder() + .partNumber(partNum).eTag(uploadResp.eTag()) + .checksumCRC32(uploadResp.checksumCRC32()).build()); + } + + s3.completeMultipartUpload(b -> b.bucket(testBucket).key(testMpuChecksumKey).uploadId(uploadId) + .multipartUpload(CompletedMultipartUpload.builder() + .parts(parts).build())); + } + + private static String generateRandomObjectKey() { + return "async-presigned-url-extension-test-" + UUID.randomUUID(); + } + + private PresignedUrlDownloadRequest createRequestForKey(String key) { + return PresignedUrlDownloadRequest.builder() + .presignedUrl(createPresignedUrl(key)) + .build(); + } + + private PresignedUrlDownloadRequest createRequestForKey(String key, String range) { + return PresignedUrlDownloadRequest.builder() + .presignedUrl(createPresignedUrl(key)) + .range(range) + .build(); + } + + private URL createPresignedUrl(String key) { + PresignedGetObjectRequest presignedRequest = presigner.presignGetObject(r -> r + .getObjectRequest(req -> req.bucket(testBucket).key(key)) + .signatureDuration(Duration.ofMinutes(10))); + return presignedRequest.url(); + } + + private String uploadTestObject(String keyPrefix, String content) { + String key = keyPrefix + "-" + UUID.randomUUID(); + S3TestUtils.putObject(AsyncPresignedUrlExtensionTestSuite.class, s3, testBucket, key, content); + + S3TestUtils.addCleanupTask(AsyncPresignedUrlExtensionTestSuite.class, () -> { + s3.deleteObject(DeleteObjectRequest.builder() + .bucket(testBucket) + .key(key) + .build()); + }); + return key; + } +} diff --git a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/crt/DefaultS3CrtAsyncClient.java b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/crt/DefaultS3CrtAsyncClient.java index 8a7bfebbff43..8ef59ac7cca8 100644 --- a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/crt/DefaultS3CrtAsyncClient.java +++ b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/crt/DefaultS3CrtAsyncClient.java @@ -80,6 +80,7 @@ import software.amazon.awssdk.services.s3.model.GetObjectResponse; import software.amazon.awssdk.services.s3.model.PutObjectRequest; import software.amazon.awssdk.services.s3.model.PutObjectResponse; +import software.amazon.awssdk.services.s3.presignedurl.AsyncPresignedUrlExtension; import software.amazon.awssdk.utils.AttributeMap; import software.amazon.awssdk.utils.CollectionUtils; import software.amazon.awssdk.utils.Validate; @@ -515,4 +516,10 @@ private static void validateCrtInClassPath() { + "on the classpath.", e); } } + + @Override + public AsyncPresignedUrlExtension presignedUrlExtension() { + // TODO: Implement presigned URL extension support for CRT client + throw new UnsupportedOperationException("Presigned URL extension is not supported for CRT client"); + } } diff --git a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/handlers/GetObjectInterceptor.java b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/handlers/GetObjectInterceptor.java index 7e13f6b97473..5929765d3b29 100644 --- a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/handlers/GetObjectInterceptor.java +++ b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/handlers/GetObjectInterceptor.java @@ -28,6 +28,7 @@ import software.amazon.awssdk.core.interceptor.SdkExecutionAttribute; import software.amazon.awssdk.core.internal.util.HttpChecksumUtils; import software.amazon.awssdk.http.SdkHttpResponse; +import software.amazon.awssdk.services.s3.internal.presignedurl.model.PresignedUrlDownloadRequestWrapper; import software.amazon.awssdk.services.s3.model.GetObjectRequest; import software.amazon.awssdk.services.s3.model.GetObjectResponse; import software.amazon.awssdk.utils.Pair; @@ -43,7 +44,8 @@ public class GetObjectInterceptor implements ExecutionInterceptor { @Override public void afterTransmission(Context.AfterTransmission context, ExecutionAttributes executionAttributes) { - if (!(context.request() instanceof GetObjectRequest)) { + if (!(context.request() instanceof GetObjectRequest) + && !(context.request() instanceof PresignedUrlDownloadRequestWrapper)) { return; } ChecksumSpecs resolvedChecksumSpecs = executionAttributes.getAttribute(SdkExecutionAttribute.RESOLVED_CHECKSUM_SPECS); diff --git a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/MultipartAsyncPresignedUrlExtension.java b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/MultipartAsyncPresignedUrlExtension.java new file mode 100644 index 000000000000..3e4b9ade0033 --- /dev/null +++ b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/MultipartAsyncPresignedUrlExtension.java @@ -0,0 +1,57 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.services.s3.internal.multipart; + +import java.util.concurrent.CompletableFuture; +import software.amazon.awssdk.annotations.SdkInternalApi; +import software.amazon.awssdk.core.async.AsyncResponseTransformer; +import software.amazon.awssdk.services.s3.S3AsyncClient; +import software.amazon.awssdk.services.s3.model.GetObjectResponse; +import software.amazon.awssdk.services.s3.presignedurl.AsyncPresignedUrlExtension; +import software.amazon.awssdk.services.s3.presignedurl.model.PresignedUrlDownloadRequest; +import software.amazon.awssdk.utils.Validate; + +/** + * An {@link AsyncPresignedUrlExtension} that automatically converts presigned URL downloads + * to multipart downloads. + */ +@SdkInternalApi +public class MultipartAsyncPresignedUrlExtension implements AsyncPresignedUrlExtension { + private final PresignedUrlDownloadHelper downloadHelper; + + public MultipartAsyncPresignedUrlExtension( + S3AsyncClient s3AsyncClient, + AsyncPresignedUrlExtension asyncPresignedUrlExtension, + long bufferSizeInBytes, + long partSizeInBytes) { + Validate.paramNotNull(s3AsyncClient, "s3AsyncClient"); + Validate.paramNotNull(asyncPresignedUrlExtension, "asyncPresignedUrlExtension"); + this.downloadHelper = new PresignedUrlDownloadHelper( + s3AsyncClient, + asyncPresignedUrlExtension, + bufferSizeInBytes, + partSizeInBytes); + } + + @Override + public CompletableFuture getObject( + PresignedUrlDownloadRequest presignedUrlDownloadRequest, + AsyncResponseTransformer asyncResponseTransformer) { + Validate.paramNotNull(presignedUrlDownloadRequest, "presignedUrlDownloadRequest"); + Validate.paramNotNull(asyncResponseTransformer, "asyncResponseTransformer"); + return downloadHelper.downloadObject(presignedUrlDownloadRequest, asyncResponseTransformer); + } +} \ No newline at end of file diff --git a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/MultipartDownloadUtils.java b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/MultipartDownloadUtils.java index 807b6a8bbbc0..a9145f1bc1dc 100644 --- a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/MultipartDownloadUtils.java +++ b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/MultipartDownloadUtils.java @@ -20,12 +20,23 @@ import java.util.Collections; import java.util.List; import java.util.Optional; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import software.amazon.awssdk.annotations.SdkInternalApi; +import software.amazon.awssdk.core.SdkResponse; +import software.amazon.awssdk.core.SplittingTransformerConfiguration; +import software.amazon.awssdk.core.async.AsyncResponseTransformer; +import software.amazon.awssdk.core.async.AsyncResponseTransformer.SplitResult; +import software.amazon.awssdk.services.s3.model.ChecksumType; import software.amazon.awssdk.services.s3.model.GetObjectRequest; +import software.amazon.awssdk.services.s3.model.GetObjectResponse; +import software.amazon.awssdk.services.s3.model.S3Request; @SdkInternalApi public final class MultipartDownloadUtils { + private static final Pattern CONTENT_RANGE_PATTERN = Pattern.compile("bytes\\s+(\\d+)-(\\d+)/(\\d+)"); + private MultipartDownloadUtils() { } @@ -58,4 +69,117 @@ public static Optional multipartDownloadResumeCo .flatMap(conf -> Optional.ofNullable(conf.executionAttributes().getAttribute(MULTIPART_DOWNLOAD_RESUME_CONTEXT))); } + /** + * This method checks the + * {@link software.amazon.awssdk.services.s3.multipart.S3MultipartExecutionAttribute#MULTIPART_DOWNLOAD_RESUME_CONTEXT} + * execution attributes for a context object and returns it if it finds one. Otherwise, returns an empty Optional. + * + * @param request the request to look for execution attributes + * @return the MultipartDownloadResumeContext if one is found, otherwise an empty Optional. + */ + public static Optional multipartDownloadResumeContext(S3Request request) { + return request + .overrideConfiguration() + .flatMap(conf -> Optional.ofNullable(conf.executionAttributes().getAttribute(MULTIPART_DOWNLOAD_RESUME_CONTEXT))); + } + + /** + * Parses start byte and end byte from a Content-Range header. + * + * @param contentRange the Content-Range header value (e.g., "bytes 0-1023/2048") + * @return array of [startByte, endByte], or null if parsing fails + */ + public static long[] parseContentRange(String contentRange) { + if (contentRange == null) { + return null; + } + Matcher matcher = CONTENT_RANGE_PATTERN.matcher(contentRange); + if (!matcher.matches()) { + return null; + } + return new long[] { + Long.parseLong(matcher.group(1)), + Long.parseLong(matcher.group(2)) + }; + } + + /** + * Parses the total size from a Content-Range header. + * + * @param contentRange the Content-Range header value (e.g., "bytes 0-1023/2048") + * @return the total size, or empty if parsing fails + */ + public static Optional parseContentRangeForTotalSize(String contentRange) { + if (contentRange == null) { + return Optional.empty(); + } + Matcher matcher = CONTENT_RANGE_PATTERN.matcher(contentRange); + if (!matcher.matches()) { + return Optional.empty(); + } + return Optional.of(Long.parseLong(matcher.group(3))); + } + + /** + * Calculates the total number of parts needed to download an object of the given size. + * + * @param contentLength total object size in bytes + * @param partSize size of each part in bytes + * @return the number of parts + */ + public static long calculateTotalParts(long contentLength, long partSize) { + return (contentLength / partSize) + (contentLength % partSize == 0 ? 0 : 1); + + } + + /** + * Rewrites a first-part response to represent the full object. + * + * @param firstPartResponse the GetObjectResponse from the first part request + * @return full-object response with total content-length, full content-range, + * and checksum values nulled if checksum type is COMPOSITE + */ + public static GetObjectResponse toFullObjectResponse(GetObjectResponse firstPartResponse) { + String contentRange = firstPartResponse.contentRange(); + Optional totalOpt = parseContentRangeForTotalSize(contentRange); + if (!totalOpt.isPresent()) { + return firstPartResponse; + } + long totalLength = totalOpt.get(); + String fullRange = "bytes 0-" + (totalLength - 1) + "/" + totalLength; + + GetObjectResponse.Builder builder = firstPartResponse.toBuilder() + .contentLength(totalLength) + .contentRange(fullRange); + + if (firstPartResponse.checksumType() == ChecksumType.COMPOSITE) { + builder.sdkFields().stream() + .filter(f -> f.memberName().startsWith("Checksum") && !"ChecksumType".equals(f.memberName())) + .forEach(f -> f.set(builder, null)); + } + + return builder.build(); + } + + /** + * Splits the given transformer with a response mapper that applies {@link #toFullObjectResponse} + * to the first part's response before it reaches the customer's transformer. + */ + public static SplitResult splitWithResponseRewrite( + AsyncResponseTransformer transformer, + SplittingTransformerConfiguration splitConfig) { + SplittingTransformerConfiguration configWithMapper = + splitConfig.toBuilder() + .responseMapper(MultipartDownloadUtils::mapToFullObjectResponse) + .build(); + return transformer.split(configWithMapper); + } + + private static SdkResponse mapToFullObjectResponse(SdkResponse response) { + if (response instanceof GetObjectResponse) { + return toFullObjectResponse((GetObjectResponse) response); + } + return response; + } + } diff --git a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/MultipartS3AsyncClient.java b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/MultipartS3AsyncClient.java index 499f36a6dd1b..6d3aca4970e1 100644 --- a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/MultipartS3AsyncClient.java +++ b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/MultipartS3AsyncClient.java @@ -35,6 +35,7 @@ import software.amazon.awssdk.services.s3.model.PutObjectResponse; import software.amazon.awssdk.services.s3.model.S3Request; import software.amazon.awssdk.services.s3.multipart.MultipartConfiguration; +import software.amazon.awssdk.services.s3.presignedurl.AsyncPresignedUrlExtension; import software.amazon.awssdk.utils.Validate; /** @@ -52,6 +53,8 @@ public final class MultipartS3AsyncClient extends DelegatingS3AsyncClient { private final CopyObjectHelper copyObjectHelper; private final DownloadObjectHelper downloadObjectHelper; private final boolean checksumEnabled; + private final long apiCallBufferSize; + private final long minPartSizeInBytes; private MultipartS3AsyncClient(S3AsyncClient delegate, MultipartConfiguration multipartConfiguration, boolean checksumEnabled) { @@ -63,6 +66,8 @@ private MultipartS3AsyncClient(S3AsyncClient delegate, MultipartConfiguration mu long threshold = resolver.thresholdInBytes(); long apiCallBufferSize = resolver.apiCallBufferSize(); int maxInFlightParts = resolver.maxInFlightParts(); + this.apiCallBufferSize = apiCallBufferSize; + this.minPartSizeInBytes = minPartSizeInBytes; mpuHelper = new UploadObjectHelper(delegate, resolver); copyObjectHelper = new CopyObjectHelper(delegate, minPartSizeInBytes, threshold); downloadObjectHelper = new DownloadObjectHelper(delegate, apiCallBufferSize, maxInFlightParts); @@ -111,4 +116,14 @@ protected CompletableFuture invokeOperat }; return new MultipartS3AsyncClient(clientWithUserAgent, multipartConfiguration, checksumEnabled); } + + @Override + public AsyncPresignedUrlExtension presignedUrlExtension() { + AsyncPresignedUrlExtension delegateExtension = ((S3AsyncClient) delegate()).presignedUrlExtension(); + return new MultipartAsyncPresignedUrlExtension( + (S3AsyncClient) delegate(), + delegateExtension, + apiCallBufferSize, + minPartSizeInBytes); + } } diff --git a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/ParallelPresignedUrlMultipartDownloaderSubscriber.java b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/ParallelPresignedUrlMultipartDownloaderSubscriber.java new file mode 100644 index 000000000000..3497aa3745e6 --- /dev/null +++ b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/ParallelPresignedUrlMultipartDownloaderSubscriber.java @@ -0,0 +1,336 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.services.s3.internal.multipart; + +import java.util.Map; +import java.util.Optional; +import java.util.Queue; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.Semaphore; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicLong; +import org.reactivestreams.Subscriber; +import org.reactivestreams.Subscription; +import software.amazon.awssdk.annotations.SdkInternalApi; +import software.amazon.awssdk.core.async.AsyncResponseTransformer; +import software.amazon.awssdk.core.exception.SdkClientException; +import software.amazon.awssdk.services.s3.S3AsyncClient; +import software.amazon.awssdk.services.s3.model.GetObjectResponse; +import software.amazon.awssdk.services.s3.presignedurl.model.PresignedUrlDownloadRequest; +import software.amazon.awssdk.utils.CompletableFutureUtils; +import software.amazon.awssdk.utils.Logger; +import software.amazon.awssdk.utils.Pair; + +/** + * A parallel subscriber for multipart presigned URL downloads that writes parts concurrently. + * Used with {@link software.amazon.awssdk.core.internal.async.FileAsyncResponseTransformerPublisher} + * when {@code parallelSplitSupported() == true} (i.e., toFile() downloads). + * + *

Unlike {@link PresignedUrlMultipartDownloaderSubscriber} which requests one part at a time, + * this subscriber requests up to {@code maxInFlightParts} concurrently, similar to + * {@link ParallelMultipartDownloaderSubscriber} for regular multipart downloads.

+ */ +@SdkInternalApi +public class ParallelPresignedUrlMultipartDownloaderSubscriber + implements Subscriber> { + + private static final Logger log = Logger.loggerFor(ParallelPresignedUrlMultipartDownloaderSubscriber.class); + + private final S3AsyncClient s3AsyncClient; + private final PresignedUrlDownloadRequest presignedUrlDownloadRequest; + private final long configuredPartSizeInBytes; + /** + * The future returned by the SplitResult, representing the overall download completion. + * Completed exceptionally on error (before cancel) to prevent ByteArraySplittingTransformer + * from assembling invalid data. + */ + private final CompletableFuture resultFuture; + private final int maxInFlightParts; + + private final AtomicLong partNumber = new AtomicLong(0); + private final AtomicLong completedParts = new AtomicLong(0); + private final Semaphore inFlightPermits; + /** + * CAS gate ensuring only the first part failure triggers error handling and cancellation. + */ + private final AtomicBoolean downloadFailed = new AtomicBoolean(false); + private final AtomicBoolean processingPending = new AtomicBoolean(false); + private final Map> inFlightRequests = new ConcurrentHashMap<>(); + private final Queue>> pendingTransformers = + new ConcurrentLinkedQueue<>(); + + private final Object subscriptionLock = new Object(); + private Subscription subscription; + + private volatile Long totalContentLength; + private volatile Long totalParts; + private volatile String eTag; + private volatile GetObjectResponse firstResponse; + + public ParallelPresignedUrlMultipartDownloaderSubscriber( + S3AsyncClient s3AsyncClient, + PresignedUrlDownloadRequest presignedUrlDownloadRequest, + long configuredPartSizeInBytes, + CompletableFuture resultFuture, + int maxInFlightParts) { + this.s3AsyncClient = s3AsyncClient; + this.presignedUrlDownloadRequest = presignedUrlDownloadRequest; + this.configuredPartSizeInBytes = configuredPartSizeInBytes; + this.resultFuture = resultFuture; + this.maxInFlightParts = maxInFlightParts; + this.inFlightPermits = new Semaphore(maxInFlightParts); + } + + @Override + public void onSubscribe(Subscription s) { + if (this.subscription != null) { + s.cancel(); + return; + } + this.subscription = s; + s.request(1); + } + + @Override + public void onNext(AsyncResponseTransformer asyncResponseTransformer) { + if (asyncResponseTransformer == null) { + throw new NullPointerException("onNext must not be called with null asyncResponseTransformer"); + } + + long currentPart = partNumber.getAndIncrement(); + + if (currentPart == 0) { + sendFirstRequest(asyncResponseTransformer); + } else { + if (totalParts != null && currentPart >= totalParts) { + return; + } + if (totalParts != null) { + processRequest(asyncResponseTransformer, currentPart); + } else { + pendingTransformers.offer(Pair.of(currentPart, asyncResponseTransformer)); + } + } + } + + private void sendFirstRequest(AsyncResponseTransformer transformer) { + PresignedUrlDownloadRequest partRequest = createRangedGetRequest(0L); + log.debug(() -> "Sending first range request with range=" + partRequest.range()); + + if (!inFlightPermits.tryAcquire()) { + throw new IllegalStateException("Failed to acquire permit for first request"); + } + + CompletableFuture response = + s3AsyncClient.presignedUrlExtension().getObject(partRequest, transformer); + + inFlightRequests.put(0L, response); + CompletableFutureUtils.forwardExceptionTo(resultFuture, response); + + response.whenComplete((res, error) -> { + inFlightRequests.remove(0L); + inFlightPermits.release(); + + if (error != null) { + if (PresignedUrlDownloadHelper.isRangeNotSatisfiable(error)) { + resultFuture.completeExceptionally( + new PresignedUrlDownloadHelper.EmptyObjectRangeNotSatisfiableException(error)); + synchronized (subscriptionLock) { + subscription.cancel(); + } + } else { + handlePartError(error, 0L); + } + return; + } + + if (downloadFailed.get()) { + return; + } + + completedParts.incrementAndGet(); + + this.eTag = res.eTag(); + this.firstResponse = res; + + String contentRange = res.contentRange(); + if (contentRange == null) { + handlePartError(PresignedUrlDownloadHelper.missingContentRangeHeader(), 0L); + return; + } + + Optional parsedTotal = MultipartDownloadUtils.parseContentRangeForTotalSize(contentRange); + if (!parsedTotal.isPresent()) { + handlePartError(PresignedUrlDownloadHelper.invalidContentRangeHeader(contentRange), 0L); + return; + } + + this.totalContentLength = parsedTotal.get(); + this.totalParts = MultipartDownloadUtils.calculateTotalParts(totalContentLength, configuredPartSizeInBytes); + log.debug(() -> String.format("Total content length: %d, Total parts: %d", totalContentLength, totalParts)); + + Optional validationError = validatePartResponse(res, 0L); + if (validationError.isPresent()) { + handlePartError(validationError.get(), 0L); + return; + } + + if (totalParts <= 1) { + resultFuture.complete(MultipartDownloadUtils.toFullObjectResponse(firstResponse)); + synchronized (subscriptionLock) { + subscription.cancel(); + } + return; + } + + processPendingTransformers(); + + long remainingParts = totalParts - 1; + long toRequest = Math.min(remainingParts, maxInFlightParts); + synchronized (subscriptionLock) { + subscription.request(toRequest); + } + }); + } + + private void processRequest(AsyncResponseTransformer transformer, + long currentPart) { + if (currentPart >= totalParts) { + return; + } + + if (!inFlightPermits.tryAcquire()) { + pendingTransformers.offer(Pair.of(currentPart, transformer)); + return; + } + + sendPartRequest(transformer, currentPart); + processPendingTransformers(); + } + + private void sendPartRequest(AsyncResponseTransformer transformer, + long partIndex) { + if (downloadFailed.get()) { + inFlightPermits.release(); + return; + } + + PresignedUrlDownloadRequest partRequest = createRangedGetRequest(partIndex); + log.debug(() -> "Sending range request for part " + partIndex + " with range=" + partRequest.range()); + + CompletableFuture response = + s3AsyncClient.presignedUrlExtension().getObject(partRequest, transformer); + + inFlightRequests.put(partIndex, response); + CompletableFutureUtils.forwardExceptionTo(resultFuture, response); + + response.whenComplete((res, error) -> { + inFlightRequests.remove(partIndex); + inFlightPermits.release(); + + if (error != null) { + handlePartError(error, partIndex); + return; + } + if (downloadFailed.get()) { + log.debug(() -> "Ignoring late completion for part " + partIndex + ", download already failed"); + return; + } + + Optional validationError = validatePartResponse(res, partIndex); + if (validationError.isPresent()) { + handlePartError(validationError.get(), partIndex); + return; + } + + log.debug(() -> "Completed part: " + partIndex); + long totalComplete = completedParts.incrementAndGet(); + + if (totalComplete == totalParts) { + resultFuture.complete(MultipartDownloadUtils.toFullObjectResponse(firstResponse)); + synchronized (subscriptionLock) { + subscription.cancel(); + } + } else { + processPendingTransformers(); + synchronized (subscriptionLock) { + subscription.request(1); + } + } + }); + } + + private void processPendingTransformers() { + // Re-check after releasing the gate to catch permits that arrived + // while exiting — prevents "missed signal" where no thread drains the queue. + do { + if (!processingPending.compareAndSet(false, true)) { + return; + } + try { + // Drain pending queue while permits are available + while (!pendingTransformers.isEmpty() && inFlightPermits.tryAcquire()) { + Pair> pendingPart = + pendingTransformers.poll(); + if (pendingPart != null && pendingPart.left() < totalParts) { + sendPartRequest(pendingPart.right(), pendingPart.left()); + } else { + inFlightPermits.release(); + } + } + } finally { + processingPending.set(false); + } + } while (!pendingTransformers.isEmpty() && inFlightPermits.availablePermits() > 0); + } + + private Optional validatePartResponse(GetObjectResponse response, long partIndex) { + return PresignedUrlDownloadHelper.validatePartResponse( + response, partIndex, configuredPartSizeInBytes, totalContentLength, totalParts); + } + + private void handlePartError(Throwable error, long partIndex) { + if (downloadFailed.compareAndSet(false, true)) { + log.debug(() -> "Error on part " + partIndex, error); + resultFuture.completeExceptionally(error); + inFlightRequests.values().forEach(future -> future.cancel(true)); + synchronized (subscriptionLock) { + if (subscription != null) { + subscription.cancel(); + } + } + } + } + + private PresignedUrlDownloadRequest createRangedGetRequest(long partIndex) { + return PresignedUrlDownloadHelper.createRangedGetRequest( + presignedUrlDownloadRequest, partIndex, configuredPartSizeInBytes, totalContentLength, eTag); + } + + @Override + public void onError(Throwable t) { + log.debug(() -> "Error in parallel multipart download", t); + resultFuture.completeExceptionally(t); + inFlightRequests.values().forEach(future -> future.cancel(true)); + } + + @Override + public void onComplete() { + // Completion is handled by resultFuture + } +} \ No newline at end of file diff --git a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/PresignedUrlDownloadHelper.java b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/PresignedUrlDownloadHelper.java new file mode 100644 index 000000000000..7b0166a250dc --- /dev/null +++ b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/PresignedUrlDownloadHelper.java @@ -0,0 +1,255 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.services.s3.internal.multipart; + +import java.util.Optional; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionException; +import software.amazon.awssdk.annotations.SdkInternalApi; +import software.amazon.awssdk.core.SplittingTransformerConfiguration; +import software.amazon.awssdk.core.async.AsyncResponseTransformer; +import software.amazon.awssdk.core.exception.SdkClientException; +import software.amazon.awssdk.services.s3.S3AsyncClient; +import software.amazon.awssdk.services.s3.model.GetObjectResponse; +import software.amazon.awssdk.services.s3.model.S3Exception; +import software.amazon.awssdk.services.s3.presignedurl.AsyncPresignedUrlExtension; +import software.amazon.awssdk.services.s3.presignedurl.model.PresignedUrlDownloadRequest; +import software.amazon.awssdk.utils.Logger; +import software.amazon.awssdk.utils.Validate; + +@SdkInternalApi +public class PresignedUrlDownloadHelper { + private static final Logger log = Logger.loggerFor(PresignedUrlDownloadHelper.class); + private static final int DEFAULT_MAX_IN_FLIGHT_PARTS = 10; + + private final S3AsyncClient s3AsyncClient; + private final AsyncPresignedUrlExtension asyncPresignedUrlExtension; + private final long bufferSizeInBytes; + private final long configuredPartSizeInBytes; + + public PresignedUrlDownloadHelper(S3AsyncClient s3AsyncClient, + AsyncPresignedUrlExtension asyncPresignedUrlExtension, + long bufferSizeInBytes, + long configuredPartSizeInBytes) { + this.s3AsyncClient = Validate.paramNotNull(s3AsyncClient, "s3AsyncClient"); + this.asyncPresignedUrlExtension = Validate.paramNotNull(asyncPresignedUrlExtension, "asyncPresignedUrlExtension"); + this.bufferSizeInBytes = Validate.isPositive(bufferSizeInBytes, "bufferSizeInBytes"); + this.configuredPartSizeInBytes = Validate.isPositive(configuredPartSizeInBytes, "configuredPartSizeInBytes"); + } + + public CompletableFuture downloadObject( + PresignedUrlDownloadRequest presignedRequest, + AsyncResponseTransformer asyncResponseTransformer) { + + Validate.paramNotNull(presignedRequest, "presignedRequest"); + Validate.paramNotNull(asyncResponseTransformer, "asyncResponseTransformer"); + + if (presignedRequest.range() != null) { + log.debug(() -> "Using single part download because presigned URL request range is included in the request. range = " + + presignedRequest.range()); + return asyncPresignedUrlExtension.getObject(presignedRequest, asyncResponseTransformer); + } + + CompletableFuture resultFuture = new CompletableFuture<>(); + doMultipartDownload(presignedRequest, asyncResponseTransformer) + .whenComplete((result, error) -> { + Throwable cause = error instanceof CompletionException ? error.getCause() : error; + // Parallel path wraps it as EmptyObjectRangeNotSatisfiableException; + // serial path (toBytes, custom transformers) surfaces raw S3Exception. + if (cause instanceof EmptyObjectRangeNotSatisfiableException + || isRangeNotSatisfiable(cause)) { + log.debug(() -> "Received 416 on first request, falling back to non-range GET for empty object"); + asyncPresignedUrlExtension.getObject(presignedRequest, asyncResponseTransformer) + .whenComplete((r, e) -> { + if (e != null) { + resultFuture.completeExceptionally(e); + } else { + resultFuture.complete(r); + } + }); + } else if (error != null) { + resultFuture.completeExceptionally(error); + } else { + resultFuture.complete(result); + } + }); + return resultFuture; + } + + private CompletableFuture doMultipartDownload( + PresignedUrlDownloadRequest presignedRequest, + AsyncResponseTransformer asyncResponseTransformer) { + + SplittingTransformerConfiguration splittingConfig = SplittingTransformerConfiguration.builder() + .bufferSizeInBytes(bufferSizeInBytes) + .build(); + + AsyncResponseTransformer.SplitResult split = + MultipartDownloadUtils.splitWithResponseRewrite(asyncResponseTransformer, splittingConfig); + + if (split.parallelSplitSupported()) { + return downloadPartsInParallel(presignedRequest, split); + } + return downloadPartsSerially(presignedRequest, split); + } + + private CompletableFuture downloadPartsInParallel( + PresignedUrlDownloadRequest presignedRequest, + AsyncResponseTransformer.SplitResult split) { + log.debug(() -> "Using parallel multipart download for presigned URL"); + ParallelPresignedUrlMultipartDownloaderSubscriber subscriber = + new ParallelPresignedUrlMultipartDownloaderSubscriber( + s3AsyncClient, + presignedRequest, + configuredPartSizeInBytes, + (CompletableFuture) split.resultFuture(), + DEFAULT_MAX_IN_FLIGHT_PARTS); + split.publisher().subscribe(subscriber); + return split.resultFuture(); + } + + private CompletableFuture downloadPartsSerially( + PresignedUrlDownloadRequest presignedRequest, + AsyncResponseTransformer.SplitResult split) { + log.debug(() -> "Using serial multipart download for presigned URL"); + PresignedUrlMultipartDownloaderSubscriber subscriber = + new PresignedUrlMultipartDownloaderSubscriber( + s3AsyncClient, + presignedRequest, + configuredPartSizeInBytes, + split.resultFuture()); + split.publisher().subscribe(subscriber); + return split.resultFuture(); + } + + static SdkClientException invalidContentRangeHeader(String contentRange) { + return SdkClientException.create("Invalid Content-Range header: " + contentRange); + } + + static SdkClientException missingContentRangeHeader() { + return SdkClientException.create("No Content-Range header in response"); + } + + static SdkClientException invalidContentLength() { + return SdkClientException.create("Invalid or missing Content-Length in response"); + } + + /** + * Validates a part response for data integrity. Checks that Content-Range and Content-Length + * match the expected values based on part index, part size, and total object size. + * + * @param response the GetObject response to validate + * @param partIndex zero-based index of this part + * @param partSizeInBytes configured part size + * @param totalContentLength total object size (from Content-Range), or null if not yet known + * @param totalParts total number of parts, or null if not yet known + * @return empty if valid, or an SdkClientException describing the mismatch + */ + static Optional validatePartResponse(GetObjectResponse response, + long partIndex, + long partSizeInBytes, + Long totalContentLength, + Long totalParts) { + String contentRange = response.contentRange(); + if (contentRange == null) { + return Optional.of(missingContentRangeHeader()); + } + + Long contentLength = response.contentLength(); + if (contentLength == null || contentLength < 0) { + return Optional.of(invalidContentLength()); + } + + long expectedStartByte = partIndex * partSizeInBytes; + long[] parsedRange = MultipartDownloadUtils.parseContentRange(contentRange); + if (parsedRange == null) { + return Optional.of(invalidContentRangeHeader(contentRange)); + } + long actualStartByte = parsedRange[0]; + long actualEndByte = parsedRange[1]; + if (actualStartByte != expectedStartByte) { + return Optional.of(SdkClientException.create( + String.format("Content-Range mismatch for part %d. Expected start byte: %d, but got: bytes %d-%d", + partIndex, expectedStartByte, actualStartByte, actualEndByte))); + } + if (totalContentLength != null) { + long expectedEndByte = Math.min(expectedStartByte + partSizeInBytes - 1, totalContentLength - 1); + if (actualEndByte != expectedEndByte) { + return Optional.of(SdkClientException.create( + String.format("Content-Range mismatch for part %d. Expected: bytes %d-%d, but got: bytes %d-%d", + partIndex, expectedStartByte, expectedEndByte, actualStartByte, actualEndByte))); + } + } + + if (totalContentLength != null && totalParts != null) { + long expectedPartSize = (partIndex == totalParts - 1) + ? totalContentLength - (partIndex * partSizeInBytes) + : partSizeInBytes; + if (!contentLength.equals(expectedPartSize)) { + return Optional.of(SdkClientException.create( + String.format("Part content length validation failed for part %d. Expected: %d, but got: %d", + partIndex, expectedPartSize, contentLength))); + } + } + return Optional.empty(); + } + + /** + * Creates a range-based GET request for a specific part of a presigned URL download. + * + * @param originalRequest the original presigned URL request + * @param partIndex zero-based index of this part + * @param partSizeInBytes configured part size + * @param totalContentLength total object size, or null if not yet known (first part) + * @param eTag ETag from first response, used for If-Match on parts 1+ + * @return a new PresignedUrlDownloadRequest with the appropriate Range and If-Match headers + */ + static PresignedUrlDownloadRequest createRangedGetRequest(PresignedUrlDownloadRequest originalRequest, + long partIndex, + long partSizeInBytes, + Long totalContentLength, + String eTag) { + long startByte = partIndex * partSizeInBytes; + long endByte = totalContentLength != null + ? Math.min(startByte + partSizeInBytes - 1, totalContentLength - 1) + : startByte + partSizeInBytes - 1; + PresignedUrlDownloadRequest.Builder builder = originalRequest.toBuilder() + .range("bytes=" + startByte + "-" + endByte); + if (partIndex > 0 && eTag != null) { + builder.ifMatch(eTag); + } + return builder.build(); + } + + /** + * Returns true if the error is a 416 Range Not Satisfiable response from S3. + * Used by subscribers to detect empty object responses on the first range request. + */ + static boolean isRangeNotSatisfiable(Throwable error) { + Throwable cause = error instanceof CompletionException ? error.getCause() : error; + return cause instanceof S3Exception && ((S3Exception) cause).statusCode() == 416; + } + + /** + * Marker exception wrapping a 416 on the first range request, signaling an empty object. + * Used to distinguish from 416 errors on subsequent requests which should propagate as failures. + */ + static class EmptyObjectRangeNotSatisfiableException extends RuntimeException { + EmptyObjectRangeNotSatisfiableException(Throwable cause) { + super("Object is empty (416 on first range request)", cause); + } + } +} \ No newline at end of file diff --git a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/PresignedUrlMultipartDownloaderSubscriber.java b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/PresignedUrlMultipartDownloaderSubscriber.java new file mode 100644 index 000000000000..86c4c76308ca --- /dev/null +++ b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/PresignedUrlMultipartDownloaderSubscriber.java @@ -0,0 +1,254 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.services.s3.internal.multipart; + +import java.util.Optional; +import java.util.Queue; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.atomic.AtomicLong; +import org.reactivestreams.Subscriber; +import org.reactivestreams.Subscription; +import software.amazon.awssdk.annotations.SdkInternalApi; +import software.amazon.awssdk.annotations.ThreadSafe; +import software.amazon.awssdk.core.async.AsyncResponseTransformer; +import software.amazon.awssdk.core.exception.SdkClientException; +import software.amazon.awssdk.services.s3.S3AsyncClient; +import software.amazon.awssdk.services.s3.model.GetObjectResponse; +import software.amazon.awssdk.services.s3.presignedurl.model.PresignedUrlDownloadRequest; +import software.amazon.awssdk.utils.Logger; + +/** + * A subscriber implementation that will download all individual parts for a multipart presigned URL download request. + * It receives individual {@link AsyncResponseTransformer} instances which will be used to perform the individual + * range-based part requests using presigned URLs. This is a 'one-shot' class, it should NOT be reused + * for more than one multipart download. + * + *

Unlike the standard {@link MultipartDownloaderSubscriber} which uses S3's native multipart API with part numbers, + * this subscriber uses HTTP range requests against presigned URLs to achieve multipart download functionality. + *

This implementation is thread-safe and handles concurrent part downloads while maintaining proper + * ordering and validation of responses.

+ */ +@ThreadSafe +@SdkInternalApi +public class PresignedUrlMultipartDownloaderSubscriber + implements Subscriber> { + + private static final Logger log = Logger.loggerFor(PresignedUrlMultipartDownloaderSubscriber.class); + + private final S3AsyncClient s3AsyncClient; + private final PresignedUrlDownloadRequest presignedUrlDownloadRequest; + private final Long configuredPartSizeInBytes; + + /** + * Internal lifecycle future for this subscriber. Completed when all parts are downloaded + * or when an error occurs. + */ + private final CompletableFuture future; + + /** + * The split transformer's completion future (from {@code SplitResult.resultFuture()}). + * Completing this signals the download result to the caller. Must be completed exceptionally + * before cancelling the subscription to prevent ByteArraySplittingTransformer from assembling + * invalid data. + */ + private final CompletableFuture resultFuture; + private final Object lock = new Object(); + private final AtomicLong nextPartIndex; + private final AtomicLong requestsSent; + + /** + * Store the GetObject futures so we can cancel them if onError() is invoked. + */ + private final Queue> getObjectFutures = new ConcurrentLinkedQueue<>(); + + private volatile Long totalContentLength; + private volatile Long totalParts; + private volatile String eTag; + private Subscription subscription; + + public PresignedUrlMultipartDownloaderSubscriber( + S3AsyncClient s3AsyncClient, + PresignedUrlDownloadRequest presignedUrlDownloadRequest, + long configuredPartSizeInBytes, + CompletableFuture resultFuture) { + this.s3AsyncClient = s3AsyncClient; + this.presignedUrlDownloadRequest = presignedUrlDownloadRequest; + this.configuredPartSizeInBytes = configuredPartSizeInBytes; + this.nextPartIndex = new AtomicLong(0); + this.requestsSent = new AtomicLong(0); + this.future = new CompletableFuture<>(); + this.resultFuture = resultFuture; + } + + @Override + public void onSubscribe(Subscription s) { + if (subscription != null) { + s.cancel(); + return; + } + this.subscription = s; + s.request(1); + } + + @Override + public void onNext(AsyncResponseTransformer asyncResponseTransformer) { + if (asyncResponseTransformer == null) { + throw new NullPointerException("onNext must not be called with null asyncResponseTransformer"); + } + + long currentPartIndex; + synchronized (lock) { + currentPartIndex = nextPartIndex.get(); + if (totalParts != null && currentPartIndex >= totalParts) { + log.debug(() -> String.format("Completing multipart download after a total of %d parts downloaded.", totalParts)); + subscription.cancel(); + return; + } + nextPartIndex.incrementAndGet(); + } + makeRangeRequest(currentPartIndex, asyncResponseTransformer); + } + + private void makeRangeRequest(long partIndex, + AsyncResponseTransformer asyncResponseTransformer) { + PresignedUrlDownloadRequest partRequest = createRangedGetRequest(partIndex); + log.debug(() -> "Sending range request for part " + partIndex + " with range=" + partRequest.range()); + + requestsSent.incrementAndGet(); + CompletableFuture responseFuture = s3AsyncClient.presignedUrlExtension() + .getObject(partRequest, asyncResponseTransformer); + getObjectFutures.add(responseFuture); + responseFuture.whenComplete((response, error) -> { + if (error != null) { + if (partIndex == 0 && PresignedUrlDownloadHelper.isRangeNotSatisfiable(error)) { + log.debug(() -> "Received 416 on first range request, object is empty"); + resultFuture.completeExceptionally( + new PresignedUrlDownloadHelper.EmptyObjectRangeNotSatisfiableException(error)); + synchronized (lock) { + subscription.cancel(); + } + } else { + handleError(error); + } + return; + } + if (validatePart(response, partIndex, asyncResponseTransformer)) { + requestMoreIfNeeded(nextPartIndex.get()); + } + }); + } + + private boolean validatePart(GetObjectResponse response, long partIndex, + AsyncResponseTransformer asyncResponseTransformer) { + long dispatched = nextPartIndex.get(); + log.debug(() -> String.format("Dispatched %d parts so far", dispatched)); + + String responseETag = response.eTag(); + String responseContentRange = response.contentRange(); + if (eTag == null) { + this.eTag = responseETag; + log.debug(() -> String.format("Multipart object ETag: %s", this.eTag)); + } + + if (totalContentLength == null && responseContentRange != null) { + Optional parsedContentLength = MultipartDownloadUtils.parseContentRangeForTotalSize(responseContentRange); + if (!parsedContentLength.isPresent()) { + SdkClientException error = PresignedUrlDownloadHelper.invalidContentRangeHeader(responseContentRange); + log.debug(() -> "Failed to parse content range", error); + asyncResponseTransformer.exceptionOccurred(error); + handleError(error); + return false; + } + + this.totalContentLength = parsedContentLength.get(); + this.totalParts = MultipartDownloadUtils.calculateTotalParts(totalContentLength, configuredPartSizeInBytes); + log.debug(() -> String.format("Total content length: %d, Total parts: %d", totalContentLength, totalParts)); + } + + Optional validationError = validateResponse(response, partIndex); + if (validationError.isPresent()) { + log.debug(() -> "Response validation failed", validationError.get()); + asyncResponseTransformer.exceptionOccurred(validationError.get()); + handleError(validationError.get()); + return false; + } + + return true; + } + + private void requestMoreIfNeeded(long dispatched) { + synchronized (lock) { + if (hasMoreParts(dispatched)) { + subscription.request(1); + } else { + if (totalParts != null && requestsSent.get() != totalParts) { + handleError(new IllegalStateException( + "Request count mismatch. Expected: " + totalParts + ", sent: " + requestsSent.get())); + return; + } + log.debug(() -> String.format("Completing multipart download after a total of %d parts downloaded.", totalParts)); + subscription.cancel(); + } + } + } + + private Optional validateResponse(GetObjectResponse response, long partIndex) { + return PresignedUrlDownloadHelper.validatePartResponse( + response, partIndex, configuredPartSizeInBytes, totalContentLength, totalParts); + } + + private boolean hasMoreParts(long dispatched) { + return totalParts != null && dispatched < totalParts; + } + + private PresignedUrlDownloadRequest createRangedGetRequest(long partIndex) { + return PresignedUrlDownloadHelper.createRangedGetRequest( + presignedUrlDownloadRequest, partIndex, configuredPartSizeInBytes, totalContentLength, eTag); + } + + private void handleError(Throwable t) { + future.completeExceptionally(t); + if (resultFuture != null) { + resultFuture.completeExceptionally(t); + } + synchronized (lock) { + if (subscription != null) { + subscription.cancel(); + } + } + } + + @Override + public void onError(Throwable t) { + log.debug(() -> "Error in multipart download", t); + CompletableFuture partFuture; + while ((partFuture = getObjectFutures.poll()) != null) { + partFuture.cancel(true); + } + future.completeExceptionally(t); + if (resultFuture != null) { + resultFuture.completeExceptionally(t); + } + } + + @Override + public void onComplete() { + future.complete(null); + } + +} \ No newline at end of file diff --git a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/presignedurl/DefaultAsyncPresignedUrlExtension.java b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/presignedurl/DefaultAsyncPresignedUrlExtension.java new file mode 100644 index 000000000000..1fcefcd98b9a --- /dev/null +++ b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/presignedurl/DefaultAsyncPresignedUrlExtension.java @@ -0,0 +1,181 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.services.s3.internal.presignedurl; + +import static software.amazon.awssdk.core.client.config.SdkClientOption.SIGNER_OVERRIDDEN; +import static software.amazon.awssdk.utils.FunctionalUtils.runAndLogError; + +import java.util.Collections; +import java.util.List; +import java.util.Optional; +import java.util.concurrent.CompletableFuture; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import software.amazon.awssdk.annotations.SdkInternalApi; +import software.amazon.awssdk.awscore.exception.AwsServiceException; +import software.amazon.awssdk.awscore.internal.AwsProtocolMetadata; +import software.amazon.awssdk.checksums.DefaultChecksumAlgorithm; +import software.amazon.awssdk.checksums.spi.ChecksumAlgorithm; +import software.amazon.awssdk.core.async.AsyncResponseTransformer; +import software.amazon.awssdk.core.async.AsyncResponseTransformerUtils; +import software.amazon.awssdk.core.client.config.SdkAdvancedClientOption; +import software.amazon.awssdk.core.client.config.SdkClientConfiguration; +import software.amazon.awssdk.core.client.config.SdkClientOption; +import software.amazon.awssdk.core.client.handler.AsyncClientHandler; +import software.amazon.awssdk.core.client.handler.ClientExecutionParams; +import software.amazon.awssdk.core.exception.SdkClientException; +import software.amazon.awssdk.core.http.HttpResponseHandler; +import software.amazon.awssdk.core.interceptor.SdkInternalExecutionAttribute; +import software.amazon.awssdk.core.interceptor.trait.HttpChecksum; +import software.amazon.awssdk.core.metrics.CoreMetric; +import software.amazon.awssdk.core.signer.NoOpSigner; +import software.amazon.awssdk.metrics.MetricCollector; +import software.amazon.awssdk.metrics.MetricPublisher; +import software.amazon.awssdk.metrics.NoOpMetricCollector; +import software.amazon.awssdk.protocols.xml.AwsS3ProtocolFactory; +import software.amazon.awssdk.protocols.xml.XmlOperationMetadata; +import software.amazon.awssdk.services.s3.internal.presignedurl.model.PresignedUrlDownloadRequestWrapper; +import software.amazon.awssdk.services.s3.model.GetObjectResponse; +import software.amazon.awssdk.services.s3.model.InvalidObjectStateException; +import software.amazon.awssdk.services.s3.model.NoSuchKeyException; +import software.amazon.awssdk.services.s3.model.S3Exception; +import software.amazon.awssdk.services.s3.presignedurl.AsyncPresignedUrlExtension; +import software.amazon.awssdk.services.s3.presignedurl.model.PresignedUrlDownloadRequest; +import software.amazon.awssdk.utils.CompletableFutureUtils; +import software.amazon.awssdk.utils.Pair; + +/** + * Default implementation of {@link AsyncPresignedUrlExtension} for executing S3 operations asynchronously using presigned URLs. + */ +@SdkInternalApi +public final class DefaultAsyncPresignedUrlExtension implements AsyncPresignedUrlExtension { + private static final Logger log = LoggerFactory.getLogger(DefaultAsyncPresignedUrlExtension.class); + + /** + * Checksum configuration matching the codegen-produced GetObject operation. + * Enables the HttpChecksumValidationInterceptor to validate response checksums. + */ + private static final HttpChecksum RESPONSE_CHECKSUM_CONFIG = + HttpChecksum.builder() + .requestValidationMode("ENABLED") + .responseAlgorithmsV2( + DefaultChecksumAlgorithm.values() + .toArray(new ChecksumAlgorithm[0])) + .build(); + + private final AsyncClientHandler clientHandler; + private final AwsS3ProtocolFactory protocolFactory; + private final SdkClientConfiguration clientConfiguration; + private final List metricPublishers; + private final AwsProtocolMetadata protocolMetadata; + + public DefaultAsyncPresignedUrlExtension(AsyncClientHandler clientHandler, + AwsS3ProtocolFactory protocolFactory, + SdkClientConfiguration clientConfiguration, + AwsProtocolMetadata protocolMetadata) { + this.clientHandler = clientHandler; + this.protocolFactory = protocolFactory; + this.protocolMetadata = protocolMetadata; + this.clientConfiguration = updateSdkClientConfiguration(clientConfiguration); + this.metricPublishers = Optional.ofNullable( + this.clientConfiguration.option(SdkClientOption.METRIC_PUBLISHERS)) + .orElse(Collections.emptyList()); + } + + @Override + public CompletableFuture getObject( + PresignedUrlDownloadRequest presignedUrlDownloadRequest, + AsyncResponseTransformer asyncResponseTransformer) + throws NoSuchKeyException, InvalidObjectStateException, + AwsServiceException, SdkClientException, S3Exception { + + PresignedUrlDownloadRequestWrapper internalRequest = PresignedUrlDownloadRequestWrapper.builder() + .url(presignedUrlDownloadRequest.presignedUrl()) + .range(presignedUrlDownloadRequest.range()) + .ifMatch(presignedUrlDownloadRequest.ifMatch()) + .build(); + + MetricCollector apiCallMetricCollector = metricPublishers.isEmpty() ? + NoOpMetricCollector.create() : MetricCollector.create("ApiCall"); + + try { + apiCallMetricCollector.reportMetric(CoreMetric.SERVICE_ID, "S3"); + apiCallMetricCollector.reportMetric(CoreMetric.OPERATION_NAME, "PresignedUrlDownload"); + + Pair, CompletableFuture> pair = + AsyncResponseTransformerUtils.wrapWithEndOfStreamFuture(asyncResponseTransformer); + AsyncResponseTransformer finalAsyncResponseTransformer = pair.left(); + asyncResponseTransformer = finalAsyncResponseTransformer; + CompletableFuture endOfStreamFuture = pair.right(); + + HttpResponseHandler responseHandler = protocolFactory.createResponseHandler( + GetObjectResponse::builder, new XmlOperationMetadata().withHasStreamingSuccessResponse(true)); + + HttpResponseHandler errorResponseHandler = protocolFactory.createErrorResponseHandler(); + + CompletableFuture executeFuture = clientHandler.execute( + new ClientExecutionParams() + .withOperationName("PresignedUrlDownload") + .withProtocolMetadata(protocolMetadata) + .withResponseHandler(responseHandler) + .withErrorResponseHandler(errorResponseHandler) + .withRequestConfiguration(clientConfiguration) + .withInput(internalRequest) + .withMetricCollector(apiCallMetricCollector) + .putExecutionAttribute(SdkInternalExecutionAttribute.SKIP_ENDPOINT_RESOLUTION, true) + .putExecutionAttribute(SdkInternalExecutionAttribute.HTTP_CHECKSUM, + checksumConfigForUrl(presignedUrlDownloadRequest.presignedUrl())) + .withMarshaller(new PresignedUrlDownloadRequestMarshaller(protocolFactory)), + asyncResponseTransformer); + + CompletableFuture whenCompleteFuture = executeFuture.whenComplete((r, e) -> { + if (e != null) { + runAndLogError(log, "Exception thrown in exceptionOccurred callback, ignoring", + () -> finalAsyncResponseTransformer.exceptionOccurred(e)); + } + endOfStreamFuture.whenComplete((r2, e2) -> { + metricPublishers.forEach(p -> p.publish(apiCallMetricCollector.collect())); + }); + }); + return CompletableFutureUtils.forwardExceptionTo(whenCompleteFuture, executeFuture); + } catch (Throwable t) { + AsyncResponseTransformer finalAsyncResponseTransformer = asyncResponseTransformer; + runAndLogError(log, "Exception thrown in exceptionOccurred callback, ignoring", + () -> finalAsyncResponseTransformer.exceptionOccurred(t)); + metricPublishers.forEach(p -> p.publish(apiCallMetricCollector.collect())); + return CompletableFutureUtils.failedFuture(t); + } + } + + private SdkClientConfiguration updateSdkClientConfiguration(SdkClientConfiguration clientConfiguration) { + SdkClientConfiguration.Builder configBuilder = clientConfiguration.toBuilder(); + configBuilder.option(SdkAdvancedClientOption.SIGNER, new NoOpSigner()); + configBuilder.option(SIGNER_OVERRIDDEN, true); + return configBuilder.build(); + } + + /** + * Returns the checksum validation config if the presigned URL was signed with checksum mode, + * or null otherwise. + */ + private static HttpChecksum checksumConfigForUrl(java.net.URL presignedUrl) { + if (PresignedUrlDownloadRequestMarshaller.hasChecksumModeInSignedHeaders(presignedUrl.getQuery())) { + return RESPONSE_CHECKSUM_CONFIG; + } + return null; + } + +} diff --git a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/presignedurl/PresignedUrlDownloadRequestMarshaller.java b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/presignedurl/PresignedUrlDownloadRequestMarshaller.java new file mode 100644 index 000000000000..b37aec335184 --- /dev/null +++ b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/presignedurl/PresignedUrlDownloadRequestMarshaller.java @@ -0,0 +1,105 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.services.s3.internal.presignedurl; + +import java.net.URI; +import software.amazon.awssdk.annotations.SdkInternalApi; +import software.amazon.awssdk.core.exception.SdkClientException; +import software.amazon.awssdk.core.runtime.transform.Marshaller; +import software.amazon.awssdk.http.SdkHttpFullRequest; +import software.amazon.awssdk.http.SdkHttpMethod; +import software.amazon.awssdk.protocols.core.OperationInfo; +import software.amazon.awssdk.protocols.core.ProtocolMarshaller; +import software.amazon.awssdk.protocols.xml.AwsXmlProtocolFactory; +import software.amazon.awssdk.services.s3.internal.presignedurl.model.PresignedUrlDownloadRequestWrapper; +import software.amazon.awssdk.utils.Validate; + +/** + * {@link PresignedUrlDownloadRequestWrapper} Marshaller + * + *

+ * Marshalls presigned URL requests by using the complete URL directly and adding optional Range headers. + * Unlike regular S3 marshalers, this preserves all embedded authentication parameters in the presigned URL. + *

+ */ +@SdkInternalApi +public class PresignedUrlDownloadRequestMarshaller implements Marshaller { + private static final OperationInfo SDK_OPERATION_BINDING = OperationInfo.builder() + .requestUri("").httpMethod(SdkHttpMethod.GET).hasExplicitPayloadMember(false).hasPayloadMembers(false) + .putAdditionalMetadata(AwsXmlProtocolFactory.ROOT_MARSHALL_LOCATION_ATTRIBUTE, null) + .putAdditionalMetadata(AwsXmlProtocolFactory.XML_NAMESPACE_ATTRIBUTE, null).build(); + + private final AwsXmlProtocolFactory protocolFactory; + + public PresignedUrlDownloadRequestMarshaller(AwsXmlProtocolFactory protocolFactory) { + this.protocolFactory = protocolFactory; + } + + /** + * Marshalls the presigned URL request into an HTTP GET request. + * + * @param presignedUrlDownloadRequestWrapper the request to marshall + * @return HTTP request ready for execution + * @throws SdkClientException if URL conversion fails + */ + @Override + public SdkHttpFullRequest marshall(PresignedUrlDownloadRequestWrapper presignedUrlDownloadRequestWrapper) { + Validate.paramNotNull(presignedUrlDownloadRequestWrapper, "presignedUrlDownloadRequestWrapper"); + try { + ProtocolMarshaller protocolMarshaller = protocolFactory + .createProtocolMarshaller(SDK_OPERATION_BINDING); + URI presignedUri = presignedUrlDownloadRequestWrapper.url().toURI(); + + SdkHttpFullRequest.Builder requestBuilder = protocolMarshaller + .marshall(presignedUrlDownloadRequestWrapper) + .toBuilder() + .uri(presignedUri); + + addChecksumModeHeaderIfSignedInUrl(requestBuilder, presignedUri); + + return requestBuilder.build(); + } catch (Exception e) { + throw SdkClientException.builder() + .message("Unable to marshall pre-signed URL Request: " + e.getMessage()) + .cause(e).build(); + } + } + + /** + * If the presigned URL's X-Amz-SignedHeaders contains "x-amz-checksum-mode", automatically add + * the header with value "ENABLED" so S3 returns checksum headers in the response. + */ + private void addChecksumModeHeaderIfSignedInUrl(SdkHttpFullRequest.Builder requestBuilder, URI uri) { + if (hasChecksumModeInSignedHeaders(uri.getQuery())) { + requestBuilder.putHeader("x-amz-checksum-mode", "ENABLED"); + } + } + + /** + * Returns true if the decoded query string's X-Amz-SignedHeaders parameter contains "x-amz-checksum-mode". + */ + static boolean hasChecksumModeInSignedHeaders(String query) { + if (query == null) { + return false; + } + for (String param : query.split("&")) { + if (param.startsWith("X-Amz-SignedHeaders=")) { + return param.substring("X-Amz-SignedHeaders=".length()).contains("x-amz-checksum-mode"); + } + } + return false; + } +} diff --git a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/presignedurl/model/PresignedUrlDownloadRequestWrapper.java b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/presignedurl/model/PresignedUrlDownloadRequestWrapper.java new file mode 100644 index 000000000000..5d5569523967 --- /dev/null +++ b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/presignedurl/model/PresignedUrlDownloadRequestWrapper.java @@ -0,0 +1,175 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.services.s3.internal.presignedurl.model; + +import java.net.URL; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.function.Function; +import software.amazon.awssdk.annotations.SdkInternalApi; +import software.amazon.awssdk.core.SdkField; +import software.amazon.awssdk.core.protocol.MarshallLocation; +import software.amazon.awssdk.core.protocol.MarshallingType; +import software.amazon.awssdk.core.traits.LocationTrait; +import software.amazon.awssdk.services.s3.model.S3Request; + +/** + * Internal request object for presigned URL GetObject operations. + *

+ * This class is used internally by the AWS SDK to process presigned URL requests for S3 GetObject operations. It contains minimal + * SdkField definitions needed for custom marshalling and is not intended for direct use by SDK users. + *

+ * Note: This is an internal implementation class and should not be used + * directly. Use {@code PresignedUrlDownloadRequest} for public API interactions. + */ +@SdkInternalApi +public final class PresignedUrlDownloadRequestWrapper extends S3Request { + private static final SdkField RANGE_FIELD = SdkField + .builder(MarshallingType.STRING) + .memberName("Range") + .getter(getter(PresignedUrlDownloadRequestWrapper::range)) + .traits(LocationTrait.builder().location(MarshallLocation.HEADER).locationName("Range") + .unmarshallLocationName("Range").build()).build(); + + private static final SdkField IF_MATCH_FIELD = SdkField + .builder(MarshallingType.STRING) + .memberName("IfMatch") + .getter(getter(PresignedUrlDownloadRequestWrapper::ifMatch)) + .traits(LocationTrait.builder().location(MarshallLocation.HEADER).locationName("If-Match") + .unmarshallLocationName("If-Match").build()).build(); + + private static final List> SDK_FIELDS = Collections.unmodifiableList( + Arrays.asList(RANGE_FIELD, IF_MATCH_FIELD)); + + private static final Map> SDK_NAME_TO_FIELD = memberNameToFieldInitializer(); + + private final URL url; + private final String range; + private final String ifMatch; + + private PresignedUrlDownloadRequestWrapper(Builder builder) { + super(builder); + this.url = builder.url; + this.range = builder.range; + this.ifMatch = builder.ifMatch; + } + + public URL url() { + return url; + } + + public String range() { + return range; + } + + public String ifMatch() { + return ifMatch; + } + + @Override + public List> sdkFields() { + return SDK_FIELDS; + } + + @Override + public Map> sdkFieldNameToField() { + return SDK_NAME_TO_FIELD; + } + + private static Function getter(Function g) { + return obj -> g.apply((PresignedUrlDownloadRequestWrapper) obj); + } + + private static Map> memberNameToFieldInitializer() { + Map> map = new HashMap<>(); + map.put("Range", RANGE_FIELD); + map.put("IfMatch", IF_MATCH_FIELD); + return Collections.unmodifiableMap(map); + } + + @Override + public Builder toBuilder() { + return new Builder(this); + } + + public static Builder builder() { + return new Builder(); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null || getClass() != obj.getClass()) { + return false; + } + if (!super.equals(obj)) { + return false; + } + PresignedUrlDownloadRequestWrapper that = (PresignedUrlDownloadRequestWrapper) obj; + return Objects.equals(url, that.url) && Objects.equals(range, that.range) && Objects.equals(ifMatch, that.ifMatch); + } + + @Override + public int hashCode() { + int result = Objects.hashCode(super.hashCode()); + result = 31 * result + Objects.hashCode(url); + result = 31 * result + Objects.hashCode(range); + result = 31 * result + Objects.hashCode(ifMatch); + return result; + } + + public static final class Builder extends S3Request.BuilderImpl { + private URL url; + private String range; + private String ifMatch; + + public Builder() { + } + + Builder(PresignedUrlDownloadRequestWrapper request) { + super(request); + this.url = request.url(); + this.range = request.range(); + this.ifMatch = request.ifMatch(); + } + + public Builder url(URL url) { + this.url = url; + return this; + } + + public Builder range(String range) { + this.range = range; + return this; + } + + public Builder ifMatch(String ifMatch) { + this.ifMatch = ifMatch; + return this; + } + + @Override + public PresignedUrlDownloadRequestWrapper build() { + return new PresignedUrlDownloadRequestWrapper(this); + } + } +} diff --git a/services/s3/src/main/java/software/amazon/awssdk/services/s3/presignedurl/AsyncPresignedUrlExtension.java b/services/s3/src/main/java/software/amazon/awssdk/services/s3/presignedurl/AsyncPresignedUrlExtension.java new file mode 100644 index 000000000000..11103a101580 --- /dev/null +++ b/services/s3/src/main/java/software/amazon/awssdk/services/s3/presignedurl/AsyncPresignedUrlExtension.java @@ -0,0 +1,132 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.services.s3.presignedurl; + +import java.nio.file.Path; +import java.util.concurrent.CompletableFuture; +import java.util.function.Consumer; +import software.amazon.awssdk.annotations.SdkPublicApi; +import software.amazon.awssdk.core.async.AsyncResponseTransformer; +import software.amazon.awssdk.core.exception.SdkClientException; +import software.amazon.awssdk.services.s3.S3AsyncClient; +import software.amazon.awssdk.services.s3.model.GetObjectResponse; +import software.amazon.awssdk.services.s3.model.S3Exception; +import software.amazon.awssdk.services.s3.presignedurl.model.PresignedUrlDownloadRequest; + +/** + * Interface for executing S3 operations asynchronously using presigned URLs. This can be accessed using + * {@link S3AsyncClient#presignedUrlExtension()}. + * + *

Checksum Validation: If the presigned URL was generated with + * {@link software.amazon.awssdk.services.s3.presigner.S3Presigner#presignGetObject} using + * {@code checksumMode(ChecksumMode.ENABLED)}, the SDK automatically sends the required header + * and S3 returns checksums for full object downloads (HTTP 200). For ranged downloads (HTTP 206), + * checksums are only returned for multipart-uploaded objects when the range aligns with original + * upload part boundaries. The downloader cannot enable checksums if the URL was not presigned + * with checksum mode. + */ +@SdkPublicApi +public interface AsyncPresignedUrlExtension { + /** + *

+ * Downloads an S3 object asynchronously using a presigned URL. + *

+ * + *

+ * This operation uses a presigned URL to download an object from Amazon S3. The presigned URL must be valid and not expired. + *

+ * + *

+ * To download a specific byte range of the object, use the range parameter in the request. + *

+ * + * @param request The presigned URL request containing the URL and optional parameters + * @param responseTransformer Transforms the response to the desired return type + * @param The type of the transformed response + * @return A {@link CompletableFuture} containing the transformed result + * @throws SdkClientException If any client side error occurs + * @throws S3Exception Base class for all S3 service exceptions + */ + default CompletableFuture getObject(PresignedUrlDownloadRequest request, + AsyncResponseTransformer responseTransformer) { + throw new UnsupportedOperationException(); + } + + /** + *

+ * Downloads an S3 object asynchronously using a presigned URL with a consumer-based request builder. + *

+ * + *

+ * This is a convenience method that creates a {@link PresignedUrlDownloadRequest} using the provided consumer. + *

+ * + * @param requestConsumer Consumer that will configure a {@link PresignedUrlDownloadRequest.Builder} + * @param responseTransformer Transforms the response to the desired return type + * @param The type of the transformed response + * @return A {@link CompletableFuture} containing the transformed result + * @throws SdkClientException If any client side error occurs + * @throws S3Exception Base class for all S3 service exceptions + */ + default CompletableFuture getObject(Consumer requestConsumer, + AsyncResponseTransformer responseTransformer) { + return getObject(PresignedUrlDownloadRequest.builder().applyMutation(requestConsumer).build(), responseTransformer); + } + + /** + *

+ * Downloads an S3 object asynchronously using a presigned URL and saves it to the specified file path. + *

+ * + *

+ * This is a convenience method that uses {@link AsyncResponseTransformer#toFile(Path)} to save the object + * directly to a file. + *

+ * + * @param request The presigned URL request containing the URL and optional parameters + * @param destinationPath The path where the downloaded object will be saved + * @return A {@link CompletableFuture} containing the {@link GetObjectResponse} + * @throws SdkClientException If any client side error occurs + * @throws S3Exception Base class for all S3 service exceptions + */ + default CompletableFuture getObject(PresignedUrlDownloadRequest request, + Path destinationPath) { + return getObject(request, AsyncResponseTransformer.toFile(destinationPath)); + } + + /** + *

+ * Downloads an S3 object asynchronously using a presigned URL with a consumer-based request builder + * and saves it to the specified file path. + *

+ * + *

+ * This is a convenience method that combines consumer-based request building with file-based response handling. + *

+ * + * @param requestConsumer Consumer that will configure a {@link PresignedUrlDownloadRequest.Builder} + * @param destinationPath The path where the downloaded object will be saved + * @return A {@link CompletableFuture} containing the {@link GetObjectResponse} + * @throws SdkClientException If any client side error occurs + * @throws S3Exception Base class for all S3 service exceptions + */ + default CompletableFuture getObject(Consumer requestConsumer, + Path destinationPath) { + return getObject(requestConsumer, AsyncResponseTransformer.toFile(destinationPath)); + } +} \ No newline at end of file diff --git a/services/s3/src/main/java/software/amazon/awssdk/services/s3/presignedurl/model/PresignedUrlDownloadRequest.java b/services/s3/src/main/java/software/amazon/awssdk/services/s3/presignedurl/model/PresignedUrlDownloadRequest.java new file mode 100644 index 000000000000..6c19159251fc --- /dev/null +++ b/services/s3/src/main/java/software/amazon/awssdk/services/s3/presignedurl/model/PresignedUrlDownloadRequest.java @@ -0,0 +1,188 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.services.s3.presignedurl.model; + +import java.net.URL; +import java.util.Objects; +import software.amazon.awssdk.annotations.SdkPublicApi; +import software.amazon.awssdk.utils.ToString; +import software.amazon.awssdk.utils.Validate; +import software.amazon.awssdk.utils.builder.CopyableBuilder; +import software.amazon.awssdk.utils.builder.ToCopyableBuilder; + +/** + * Request object for performing download operations using a presigned URL. + */ +@SdkPublicApi +public final class PresignedUrlDownloadRequest implements ToCopyableBuilder { + private final URL presignedUrl; + private final String range; + private final String ifMatch; + + private PresignedUrlDownloadRequest(BuilderImpl builder) { + this.presignedUrl = builder.presignedUrl; + this.range = builder.range; + this.ifMatch = builder.ifMatch; + } + + /** + *

+ * The presigned URL for the S3 object. This URL contains all necessary authentication information and can be used to download + * the object without additional credentials. + *

+ * Note: Presigned URLs have a limited lifetime and will expire after the + * specified duration. Ensure the URL is used before expiration. + * + * @return The presigned URL for the S3 object + */ + public URL presignedUrl() { + return presignedUrl; + } + + /** + *

+ * Specifies the byte range of an object. For more information about the HTTP Range header, see + * + * https://www.rfc-editor.org/rfc/rfc9110.html#name-range. + *

+ * Note: Amazon S3 doesn't support retrieving multiple ranges of data per GET request. + * + * @return The HTTP Range header value, or null if not specified. + */ + public String range() { + return range; + } + + /** + *

+ * Return the object only if its entity tag (ETag) is the same as the one specified in this header, + * otherwise return a 412 (precondition failed) error. + *

+ * + * @return The If-Match header value, or null if not specified. + */ + public String ifMatch() { + return ifMatch; + } + + @Override + public Builder toBuilder() { + return new BuilderImpl(this); + } + + public static Builder builder() { + return new BuilderImpl(); + } + + public static Class serializableBuilderClass() { + return BuilderImpl.class; + } + + @Override + public int hashCode() { + int hashCode = 1; + hashCode = 31 * hashCode + Objects.hashCode(presignedUrl()); + hashCode = 31 * hashCode + Objects.hashCode(range()); + hashCode = 31 * hashCode + Objects.hashCode(ifMatch()); + return hashCode; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null || getClass() != obj.getClass()) { + return false; + } + PresignedUrlDownloadRequest other = (PresignedUrlDownloadRequest) obj; + return Objects.equals(presignedUrl(), other.presignedUrl()) && + Objects.equals(range(), other.range()) && + Objects.equals(ifMatch(), other.ifMatch()); + } + + @Override + public String toString() { + return ToString.builder("PresignedUrlDownloadRequest") + .add("PresignedUrl", presignedUrl()) + .add("Range", range()) + .add("IfMatch", ifMatch()) + .build(); + } + + public interface Builder extends CopyableBuilder { + /** + * Sets the presigned URL for the S3 object. + * @param presignedUrl + * @return Returns a reference to this object so that method calls can be chained together. + */ + Builder presignedUrl(URL presignedUrl); + + /** + * Specifies the byte range of an object. + * @param range The HTTP Range header value (e.g., "bytes=0-1023") + * @return Returns a reference to this object so that method calls can be chained together. + */ + Builder range(String range); + + /** + * Return the object only if its entity tag (ETag) is the same as the one specified in this header. + * @param ifMatch The If-Match header value (ETag) + * @return Returns a reference to this object so that method calls can be chained together. + */ + Builder ifMatch(String ifMatch); + } + + static final class BuilderImpl implements Builder { + private URL presignedUrl; + private String range; + private String ifMatch; + + private BuilderImpl() { + } + + private BuilderImpl(PresignedUrlDownloadRequest presignedUrlDownloadRequest) { + presignedUrl(presignedUrlDownloadRequest.presignedUrl()); + range(presignedUrlDownloadRequest.range()); + ifMatch(presignedUrlDownloadRequest.ifMatch()); + } + + @Override + public Builder presignedUrl(URL presignedUrl) { + this.presignedUrl = presignedUrl; + return this; + } + + @Override + public Builder range(String range) { + this.range = range; + return this; + } + + @Override + public Builder ifMatch(String ifMatch) { + this.ifMatch = ifMatch; + return this; + } + + @Override + public PresignedUrlDownloadRequest build() { + Validate.paramNotNull(presignedUrl, "presignedUrl"); + return new PresignedUrlDownloadRequest(this); + } + } +} diff --git a/services/s3/src/main/java/software/amazon/awssdk/services/s3/presigner/S3Presigner.java b/services/s3/src/main/java/software/amazon/awssdk/services/s3/presigner/S3Presigner.java index e23516178e92..b5fd634255fa 100644 --- a/services/s3/src/main/java/software/amazon/awssdk/services/s3/presigner/S3Presigner.java +++ b/services/s3/src/main/java/software/amazon/awssdk/services/s3/presigner/S3Presigner.java @@ -294,6 +294,12 @@ static Builder builder() { * signing or authentication. *

* + * Checksum support: Setting {@code checksumMode(ChecksumMode.ENABLED)} on the + * {@code GetObjectRequest} enables the downloader to receive checksums from S3 for data integrity + * validation. The resulting URL will not be browser-executable (requires the + * {@code x-amz-checksum-mode} header at download time, which the SDK sends automatically). + *

+ * * Example Usage *

* diff --git a/services/s3/src/main/resources/codegen-resources/customization.config b/services/s3/src/main/resources/codegen-resources/customization.config index 1b772237a1dc..83abac701f09 100644 --- a/services/s3/src/main/resources/codegen-resources/customization.config +++ b/services/s3/src/main/resources/codegen-resources/customization.config @@ -359,5 +359,6 @@ "methodName": "modifyUploadPartCopyRequest", "className": "software.amazon.awssdk.services.s3.internal.CustomRequestTransformerUtils" } - } + }, + "presignedUrlExtensionSupported": true } diff --git a/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/multipart/MultipartDownloadUtilsTest.java b/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/multipart/MultipartDownloadUtilsTest.java index 5fce3aff2144..b88c6cd8a33d 100644 --- a/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/multipart/MultipartDownloadUtilsTest.java +++ b/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/multipart/MultipartDownloadUtilsTest.java @@ -19,7 +19,10 @@ import static software.amazon.awssdk.services.s3.multipart.S3MultipartExecutionAttribute.MULTIPART_DOWNLOAD_RESUME_CONTEXT; import org.junit.jupiter.api.Test; +import software.amazon.awssdk.http.SdkHttpResponse; +import software.amazon.awssdk.services.s3.model.ChecksumType; import software.amazon.awssdk.services.s3.model.GetObjectRequest; +import software.amazon.awssdk.services.s3.model.GetObjectResponse; class MultipartDownloadUtilsTest { @@ -60,4 +63,116 @@ void contextWithParts_contextShouldBePresent() { assertThat(MultipartDownloadUtils.multipartDownloadResumeContext(req)).isPresent(); } -} \ No newline at end of file + @Test + void parseContentRange_shouldParseValidAndInvalidRanges() { + long[] result = MultipartDownloadUtils.parseContentRange("bytes 0-1023/2048"); + assertThat(result).isNotNull(); + assertThat(result[0]).isEqualTo(0); + assertThat(result[1]).isEqualTo(1023); + + result = MultipartDownloadUtils.parseContentRange("bytes 1024-2047/2048"); + assertThat(result[0]).isEqualTo(1024); + assertThat(result[1]).isEqualTo(2047); + + assertThat(MultipartDownloadUtils.parseContentRange("invalid")).isNull(); + assertThat(MultipartDownloadUtils.parseContentRange(null)).isNull(); + } + + @Test + void parseContentRangeForTotalSize_shouldParseValidAndInvalidRanges() { + assertThat(MultipartDownloadUtils.parseContentRangeForTotalSize("bytes 0-1023/2048")).contains(2048L); + assertThat(MultipartDownloadUtils.parseContentRangeForTotalSize("bytes 0-0/1")).contains(1L); + + assertThat(MultipartDownloadUtils.parseContentRangeForTotalSize("invalid")).isEmpty(); + assertThat(MultipartDownloadUtils.parseContentRangeForTotalSize(null)).isEmpty(); + } + + @Test + void calculateTotalParts_shouldCalculateCorrectly() { + assertThat(MultipartDownloadUtils.calculateTotalParts(32, 16)).isEqualTo(2); // exact fit + assertThat(MultipartDownloadUtils.calculateTotalParts(33, 16)).isEqualTo(3); // remainder rounds up + assertThat(MultipartDownloadUtils.calculateTotalParts(1, 16)).isEqualTo(1); // smaller than part size + assertThat(MultipartDownloadUtils.calculateTotalParts(16, 16)).isEqualTo(1); // exactly one part + assertThat(MultipartDownloadUtils.calculateTotalParts(0, 16)).isEqualTo(0); // empty object + // 5 GiB / 1 byte = 5_368_709_120 parts (exceeds Integer.MAX_VALUE) + long fiveGiB = 5L * 1024L * 1024L * 1024L; + assertThat(MultipartDownloadUtils.calculateTotalParts(fiveGiB, 1L)).isEqualTo(fiveGiB); + assertThat(MultipartDownloadUtils.calculateTotalParts(Long.MAX_VALUE - 1, 1L)) + .isEqualTo(Long.MAX_VALUE - 1); + assertThat(MultipartDownloadUtils.calculateTotalParts(Long.MAX_VALUE, 2)) + .isEqualTo((Long.MAX_VALUE / 2) + 1); + } + + @Test + void toFullObjectResponse_setsContentLengthAndContentRange() { + GetObjectResponse response = GetObjectResponse.builder() + .contentLength(1024L) + .contentRange("bytes 0-1023/4096") + .eTag("\"abc123\"") + .build(); + response = setHttpResponse(response, 206, "Partial Content"); + + GetObjectResponse result = MultipartDownloadUtils.toFullObjectResponse(response); + + assertThat(result.contentLength()).isEqualTo(4096L); + assertThat(result.contentRange()).isEqualTo("bytes 0-4095/4096"); + assertThat(result.eTag()).isEqualTo("\"abc123\""); + } + + @Test + void toFullObjectResponse_noContentRange_returnsUnchanged() { + GetObjectResponse response = GetObjectResponse.builder() + .contentLength(1024L) + .build(); + response = setHttpResponse(response, 206, "Partial Content"); + + GetObjectResponse result = MultipartDownloadUtils.toFullObjectResponse(response); + + assertThat(result.contentLength()).isEqualTo(1024L); + } + + @Test + void toFullObjectResponse_compositeChecksum_nullsChecksumValues() { + GetObjectResponse response = GetObjectResponse.builder() + .contentLength(8388608L) + .contentRange("bytes 0-8388607/25165824") + .checksumType(ChecksumType.COMPOSITE) + .checksumCRC32("/g/pWA==") + .build(); + response = setHttpResponse(response, 206, "Partial Content"); + + GetObjectResponse result = MultipartDownloadUtils.toFullObjectResponse(response); + + assertThat(result.checksumType()).isEqualTo(ChecksumType.COMPOSITE); + assertThat(result.checksumCRC32()).isNull(); + assertThat(result.checksumCRC32C()).isNull(); + assertThat(result.checksumCRC64NVME()).isNull(); + assertThat(result.checksumSHA1()).isNull(); + assertThat(result.checksumSHA256()).isNull(); + assertThat(result.contentLength()).isEqualTo(25165824L); + } + + @Test + void toFullObjectResponse_fullObjectChecksum_preservesChecksumValues() { + GetObjectResponse response = GetObjectResponse.builder() + .contentLength(8388608L) + .contentRange("bytes 0-8388607/25165824") + .checksumType(ChecksumType.FULL_OBJECT) + .checksumCRC32("abc123") + .build(); + response = setHttpResponse(response, 206, "Partial Content"); + + GetObjectResponse result = MultipartDownloadUtils.toFullObjectResponse(response); + + assertThat(result.checksumType()).isEqualTo(ChecksumType.FULL_OBJECT); + assertThat(result.checksumCRC32()).isEqualTo("abc123"); + } + + private static GetObjectResponse setHttpResponse(GetObjectResponse response, int statusCode, String statusText) { + SdkHttpResponse httpResponse = SdkHttpResponse.builder() + .statusCode(statusCode) + .statusText(statusText) + .build(); + return (GetObjectResponse) response.toBuilder().sdkHttpResponse(httpResponse).build(); + } +} diff --git a/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/multipart/MultipartS3AsyncClientTest.java b/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/multipart/MultipartS3AsyncClientTest.java index 848d7259124b..d92fd69ac3c9 100644 --- a/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/multipart/MultipartS3AsyncClientTest.java +++ b/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/multipart/MultipartS3AsyncClientTest.java @@ -15,13 +15,17 @@ package software.amazon.awssdk.services.s3.internal.multipart; +import static org.assertj.core.api.AssertionsForClassTypes.assertThat; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; +import java.net.MalformedURLException; +import java.net.URL; import org.junit.jupiter.api.Test; import software.amazon.awssdk.core.ResponseBytes; import software.amazon.awssdk.core.SplittingTransformerConfiguration; @@ -30,6 +34,8 @@ import software.amazon.awssdk.services.s3.model.GetObjectRequest; import software.amazon.awssdk.services.s3.model.GetObjectResponse; import software.amazon.awssdk.services.s3.multipart.MultipartConfiguration; +import software.amazon.awssdk.services.s3.presignedurl.AsyncPresignedUrlExtension; +import software.amazon.awssdk.services.s3.presignedurl.model.PresignedUrlDownloadRequest; class MultipartS3AsyncClientTest { @@ -64,4 +70,52 @@ void partManuallySpecified_shouldBypassMultipart() { verify(mockTransformer, never()).split(any(SplittingTransformerConfiguration.class)); verify(mockDelegate, times(1)).getObject(any(GetObjectRequest.class), eq(mockTransformer)); } + + @Test + void presignedUrlExtension_rangeSpecified_shouldBypassMultipart() throws MalformedURLException { + S3AsyncClient mockDelegate = mock(S3AsyncClient.class); + AsyncPresignedUrlExtension mockDelegateExtension = mock(AsyncPresignedUrlExtension.class); + AsyncResponseTransformer mockTransformer = mock(AsyncResponseTransformer.class); + PresignedUrlDownloadRequest req = PresignedUrlDownloadRequest.builder() + .presignedUrl(new URL("https://s3.amazonaws.com/bucket/key?signature=abc")) + .range("bytes=0-1023") + .build(); + when(mockDelegate.presignedUrlExtension()).thenReturn(mockDelegateExtension); + S3AsyncClient s3AsyncClient = MultipartS3AsyncClient.create(mockDelegate, MultipartConfiguration.builder().build(), true); + s3AsyncClient.presignedUrlExtension().getObject(req, mockTransformer); + verify(mockTransformer, never()).split(any(SplittingTransformerConfiguration.class)); + verify(mockDelegateExtension, times(1)).getObject(eq(req), eq(mockTransformer)); + } + + // TODO: Enable this test once PresignedUrlMultipartDownloaderSubscriber is implemented + // // Currently fails because multipart presigned URL download throws UnsupportedOperationException + // @Test + // void presignedUrlExtension_noRange_shouldUseMultipart() throws MalformedURLException { + // S3AsyncClient mockDelegate = mock(S3AsyncClient.class); + // AsyncPresignedUrlExtension mockDelegateExtension = mock(AsyncPresignedUrlExtension.class); + // AsyncResponseTransformer mockTransformer = mock(AsyncResponseTransformer.class); + // AsyncResponseTransformer.SplitResult mockSplitResult = mock(AsyncResponseTransformer.SplitResult.class); + // PresignedUrlDownloadRequest req = PresignedUrlDownloadRequest.builder() + // .presignedUrl(new URL("https://s3.amazonaws.com/bucket/key?signature=abc")) + // .build(); + // when(mockDelegate.presignedUrlExtension()).thenReturn(mockDelegateExtension); + // when(mockTransformer.split(any(SplittingTransformerConfiguration.class))).thenReturn(mockSplitResult); + // when(mockSplitResult.publisher()).thenReturn(mock(software.amazon.awssdk.core.async.SdkPublisher.class)); + // S3AsyncClient s3AsyncClient = MultipartS3AsyncClient.create(mockDelegate, MultipartConfiguration.builder().build(), true); + // s3AsyncClient.presignedUrlExtension().getObject(req, mockTransformer); + // verify(mockTransformer, times(1)).split(any(SplittingTransformerConfiguration.class)); + // verify(mockDelegateExtension, never()).getObject(any(PresignedUrlDownloadRequest.class), any(AsyncResponseTransformer.class)); + // } + + @Test + void presignedUrlExtension_shouldReturnMultipartExtension() { + S3AsyncClient mockDelegate = mock(S3AsyncClient.class); + AsyncPresignedUrlExtension mockDelegateExtension = mock(AsyncPresignedUrlExtension.class); + when(mockDelegate.presignedUrlExtension()).thenReturn(mockDelegateExtension); + + S3AsyncClient s3AsyncClient = MultipartS3AsyncClient.create(mockDelegate, MultipartConfiguration.builder().build(), true); + AsyncPresignedUrlExtension extension = s3AsyncClient.presignedUrlExtension(); + + assertThat(extension).isInstanceOf(MultipartAsyncPresignedUrlExtension.class); + } } diff --git a/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/multipart/ParallelPresignedUrlMultipartDownloaderSubscriberTckTest.java b/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/multipart/ParallelPresignedUrlMultipartDownloaderSubscriberTckTest.java new file mode 100644 index 000000000000..bb05a0833a2e --- /dev/null +++ b/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/multipart/ParallelPresignedUrlMultipartDownloaderSubscriberTckTest.java @@ -0,0 +1,134 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.services.s3.internal.multipart; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.when; + +import java.net.MalformedURLException; +import java.nio.ByteBuffer; +import java.util.concurrent.CompletableFuture; +import org.mockito.Mockito; +import org.reactivestreams.Subscriber; +import org.reactivestreams.Subscription; +import org.reactivestreams.tck.SubscriberWhiteboxVerification; +import org.reactivestreams.tck.TestEnvironment; +import software.amazon.awssdk.core.async.AsyncResponseTransformer; +import software.amazon.awssdk.core.async.SdkPublisher; +import software.amazon.awssdk.services.s3.S3AsyncClient; +import software.amazon.awssdk.services.s3.model.GetObjectResponse; +import software.amazon.awssdk.services.s3.presignedurl.AsyncPresignedUrlExtension; +import software.amazon.awssdk.services.s3.presignedurl.model.PresignedUrlDownloadRequest; + +public class ParallelPresignedUrlMultipartDownloaderSubscriberTckTest + extends SubscriberWhiteboxVerification> { + + private final S3AsyncClient s3mock; + + public ParallelPresignedUrlMultipartDownloaderSubscriberTckTest() { + super(new TestEnvironment()); + this.s3mock = Mockito.mock(S3AsyncClient.class); + AsyncPresignedUrlExtension presignedUrlExtension = Mockito.mock(AsyncPresignedUrlExtension.class); + when(s3mock.presignedUrlExtension()).thenReturn(presignedUrlExtension); + + when(presignedUrlExtension.getObject(any(PresignedUrlDownloadRequest.class), any(AsyncResponseTransformer.class))) + .thenReturn(CompletableFuture.completedFuture( + GetObjectResponse.builder() + .contentRange("bytes 0-8388607/33554432") + .contentLength(8388608L) + .eTag("\"test-etag\"") + .build())); + } + + @Override + public Subscriber> createSubscriber( + WhiteboxSubscriberProbe> probe) { + + return new ParallelPresignedUrlMultipartDownloaderSubscriber( + s3mock, + createTestPresignedUrlRequest(), + 8 * 1024 * 1024L, + new CompletableFuture<>(), + 10 + ) { + @Override + public void onSubscribe(Subscription s) { + super.onSubscribe(s); + probe.registerOnSubscribe(new SubscriberPuppet() { + @Override + public void triggerRequest(long elements) { + s.request(elements); + } + + @Override + public void signalCancel() { + s.cancel(); + } + }); + } + + @Override + public void onNext(AsyncResponseTransformer item) { + super.onNext(item); + probe.registerOnNext(item); + } + + @Override + public void onError(Throwable t) { + super.onError(t); + probe.registerOnError(t); + } + + @Override + public void onComplete() { + super.onComplete(); + probe.registerOnComplete(); + } + }; + } + + @Override + public AsyncResponseTransformer createElement(int element) { + return new AsyncResponseTransformer() { + @Override + public CompletableFuture prepare() { + return new CompletableFuture<>(); + } + + @Override + public void onResponse(GetObjectResponse response) { + } + + @Override + public void onStream(SdkPublisher publisher) { + } + + @Override + public void exceptionOccurred(Throwable error) { + } + }; + } + + private PresignedUrlDownloadRequest createTestPresignedUrlRequest() { + try { + return PresignedUrlDownloadRequest.builder() + .presignedUrl(java.net.URI.create("https://test-bucket.s3.amazonaws.com/test-key").toURL()) + .build(); + } catch (MalformedURLException e) { + throw new RuntimeException(e); + } + } +} diff --git a/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/multipart/ParallelPresignedUrlMultipartDownloaderSubscriberTest.java b/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/multipart/ParallelPresignedUrlMultipartDownloaderSubscriberTest.java new file mode 100644 index 000000000000..57ace09759f0 --- /dev/null +++ b/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/multipart/ParallelPresignedUrlMultipartDownloaderSubscriberTest.java @@ -0,0 +1,430 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.services.s3.internal.multipart; + +import static com.github.tomakehurst.wiremock.client.WireMock.aResponse; +import static com.github.tomakehurst.wiremock.client.WireMock.absent; +import static com.github.tomakehurst.wiremock.client.WireMock.get; +import static com.github.tomakehurst.wiremock.client.WireMock.getRequestedFor; +import static com.github.tomakehurst.wiremock.client.WireMock.matching; +import static com.github.tomakehurst.wiremock.client.WireMock.stubFor; +import static com.github.tomakehurst.wiremock.client.WireMock.urlEqualTo; +import static com.github.tomakehurst.wiremock.client.WireMock.verify; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import com.github.tomakehurst.wiremock.junit5.WireMockRuntimeInfo; +import com.github.tomakehurst.wiremock.junit5.WireMockTest; +import java.io.IOException; +import java.net.MalformedURLException; +import java.net.URI; +import java.net.URL; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Arrays; +import java.util.UUID; +import java.util.concurrent.CompletionException; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import software.amazon.awssdk.core.async.AsyncResponseTransformer; +import software.amazon.awssdk.core.exception.SdkClientException; +import software.amazon.awssdk.services.s3.S3AsyncClient; +import software.amazon.awssdk.services.s3.model.GetObjectResponse; +import software.amazon.awssdk.services.s3.multipart.MultipartConfiguration; +import software.amazon.awssdk.services.s3.presignedurl.model.PresignedUrlDownloadRequest; + +/** + * Unit tests for {@link ParallelPresignedUrlMultipartDownloaderSubscriber}. + * Tests parallel-specific behavior: single-part detection, concurrent requests, + * and error propagation with in-flight cancellation. + */ +@WireMockTest +class ParallelPresignedUrlMultipartDownloaderSubscriberTest { + + private static final String PRESIGNED_URL_PATH = "/parallel-test"; + private static final byte[] TEST_DATA = "ABCDEFGHIJKLMNOPQRSTUVWXYZ123456".getBytes(StandardCharsets.UTF_8); // 32 bytes + + private S3AsyncClient s3AsyncClient; + private URL presignedUrl; + private Path tempFile; + + @BeforeEach + void setup(WireMockRuntimeInfo wiremock) throws MalformedURLException { + MultipartConfiguration multipartConfig = MultipartConfiguration.builder() + .minimumPartSizeInBytes(16L) + .build(); + s3AsyncClient = S3AsyncClient.builder() + .endpointOverride(URI.create("http://localhost:" + wiremock.getHttpPort())) + .multipartEnabled(true) + .multipartConfiguration(multipartConfig) + .build(); + presignedUrl = new URL("http://localhost:" + wiremock.getHttpPort() + PRESIGNED_URL_PATH); + } + + @AfterEach + void cleanup() throws IOException { + if (tempFile != null && Files.exists(tempFile)) { + Files.delete(tempFile); + } + } + + @Test + void singlePartObject_shouldCompleteWithoutAdditionalRequests() throws IOException { + byte[] smallData = "0123456789".getBytes(StandardCharsets.UTF_8); + stubFor(get(urlEqualTo(PRESIGNED_URL_PATH)) + .withHeader("Range", matching("bytes=0-15")) + .willReturn(aResponse() + .withStatus(206) + .withHeader("Content-Length", String.valueOf(smallData.length)) + .withHeader("Content-Range", "bytes 0-9/10") + .withHeader("ETag", "\"single-part-etag\"") + .withBody(smallData))); + + tempFile = createTempFile(); + PresignedUrlDownloadRequest request = PresignedUrlDownloadRequest.builder() + .presignedUrl(presignedUrl) + .build(); + GetObjectResponse response = (GetObjectResponse) s3AsyncClient.presignedUrlExtension() + .getObject(request, AsyncResponseTransformer.toFile(tempFile)) + .join(); + + assertThat(response.eTag()).isEqualTo("\"single-part-etag\""); + assertThat(Files.readAllBytes(tempFile)).isEqualTo(smallData); + verify(1, getRequestedFor(urlEqualTo(PRESIGNED_URL_PATH))); + } + + @Test + void multiPartObject_shouldDownloadAllPartsConcurrently() throws IOException { + stubFor(get(urlEqualTo(PRESIGNED_URL_PATH)) + .withHeader("Range", matching("bytes=0-15")) + .willReturn(aResponse() + .withStatus(206) + .withHeader("Content-Length", "16") + .withHeader("Content-Range", "bytes 0-15/32") + .withHeader("ETag", "\"multi-etag\"") + .withBody(Arrays.copyOfRange(TEST_DATA, 0, 16)))); + + stubFor(get(urlEqualTo(PRESIGNED_URL_PATH)) + .withHeader("Range", matching("bytes=16-31")) + .willReturn(aResponse() + .withStatus(206) + .withHeader("Content-Length", "16") + .withHeader("Content-Range", "bytes 16-31/32") + .withHeader("ETag", "\"multi-etag\"") + .withBody(Arrays.copyOfRange(TEST_DATA, 16, 32)))); + + tempFile = createTempFile(); + PresignedUrlDownloadRequest request = PresignedUrlDownloadRequest.builder() + .presignedUrl(presignedUrl) + .build(); + s3AsyncClient.presignedUrlExtension() + .getObject(request, AsyncResponseTransformer.toFile(tempFile)) + .join(); + + assertThat(Files.readAllBytes(tempFile)).isEqualTo(TEST_DATA); + } + + @Test + void errorOnSecondPart_shouldCompleteExceptionallyAndNotSendMoreRequests() { + stubFor(get(urlEqualTo(PRESIGNED_URL_PATH)) + .withHeader("Range", matching("bytes=0-15")) + .willReturn(aResponse() + .withStatus(206) + .withHeader("Content-Length", "16") + .withHeader("Content-Range", "bytes 0-15/48") + .withHeader("ETag", "\"error-etag\"") + .withBody(Arrays.copyOfRange(TEST_DATA, 0, 16)))); + + stubFor(get(urlEqualTo(PRESIGNED_URL_PATH)) + .withHeader("Range", matching("bytes=16-31")) + .willReturn(aResponse() + .withStatus(500) + .withBody("InternalError" + + "Simulated failure"))); + + tempFile = createTempFileUnchecked(); + PresignedUrlDownloadRequest request = PresignedUrlDownloadRequest.builder() + .presignedUrl(presignedUrl) + .build(); + + assertThatThrownBy(() -> s3AsyncClient.presignedUrlExtension() + .getObject(request, AsyncResponseTransformer.toFile(tempFile)) + .join()) + .isInstanceOf(CompletionException.class); + } + + @Test + void missingContentRangeOnFirstPart_shouldFail() { + stubFor(get(urlEqualTo(PRESIGNED_URL_PATH)) + .willReturn(aResponse() + .withStatus(206) + .withHeader("Content-Length", "16") + .withHeader("ETag", "\"no-range-etag\"") + .withBody(Arrays.copyOfRange(TEST_DATA, 0, 16)))); + + tempFile = createTempFileUnchecked(); + PresignedUrlDownloadRequest request = PresignedUrlDownloadRequest.builder() + .presignedUrl(presignedUrl) + .build(); + + assertThatThrownBy(() -> s3AsyncClient.presignedUrlExtension() + .getObject(request, AsyncResponseTransformer.toFile(tempFile)) + .join()) + .hasRootCauseInstanceOf(SdkClientException.class) + .hasMessageContaining("No Content-Range header"); + } + + @Test + void contentRangeMismatchOnSecondPart_shouldFail() { + stubFor(get(urlEqualTo(PRESIGNED_URL_PATH)) + .withHeader("Range", matching("bytes=0-15")) + .willReturn(aResponse() + .withStatus(206) + .withHeader("Content-Length", "16") + .withHeader("Content-Range", "bytes 0-15/32") + .withHeader("ETag", "\"mismatch-etag\"") + .withBody(Arrays.copyOfRange(TEST_DATA, 0, 16)))); + + stubFor(get(urlEqualTo(PRESIGNED_URL_PATH)) + .withHeader("Range", matching("bytes=16-31")) + .willReturn(aResponse() + .withStatus(206) + .withHeader("Content-Length", "16") + .withHeader("Content-Range", "bytes 9999-10014/32") + .withHeader("ETag", "\"mismatch-etag\"") + .withBody(Arrays.copyOfRange(TEST_DATA, 16, 32)))); + + tempFile = createTempFileUnchecked(); + PresignedUrlDownloadRequest request = PresignedUrlDownloadRequest.builder() + .presignedUrl(presignedUrl) + .build(); + + assertThatThrownBy(() -> s3AsyncClient.presignedUrlExtension() + .getObject(request, AsyncResponseTransformer.toFile(tempFile)) + .join()) + .hasRootCauseInstanceOf(SdkClientException.class) + .hasMessageContaining("Content-Range mismatch"); + } + + @Test + void onNext_withNullTransformer_shouldThrowNPE() { + ParallelPresignedUrlMultipartDownloaderSubscriber subscriber = + new ParallelPresignedUrlMultipartDownloaderSubscriber( + s3AsyncClient, + PresignedUrlDownloadRequest.builder().presignedUrl(presignedUrl).build(), + 16L, + new java.util.concurrent.CompletableFuture<>(), + 10); + + assertThatThrownBy(() -> subscriber.onNext(null)) + .isInstanceOf(NullPointerException.class); + } + + @Test + void multiPartDownload_manyParts_shouldCompleteSuccessfully() throws Exception { + // 13 parts to exceed maxInFlightParts (10) + byte[] data = new byte[208]; // 13 × 16 bytes + Arrays.fill(data, (byte) 'X'); + + stubFor(get(urlEqualTo(PRESIGNED_URL_PATH)) + .withHeader("Range", matching("bytes=0-15")) + .willReturn(aResponse().withStatus(206) + .withHeader("Content-Length", "16") + .withHeader("Content-Range", "bytes 0-15/208") + .withHeader("ETag", "\"etag\"") + .withBody(Arrays.copyOfRange(data, 0, 16)))); + + for (int i = 1; i < 13; i++) { + int start = i * 16; + int end = start + 15; + stubFor(get(urlEqualTo(PRESIGNED_URL_PATH)) + .withHeader("Range", matching("bytes=" + start + "-" + end)) + .willReturn(aResponse().withStatus(206) + .withHeader("Content-Length", "16") + .withHeader("Content-Range", "bytes " + start + "-" + end + "/208") + .withHeader("ETag", "\"etag\"") + .withBody(Arrays.copyOfRange(data, start, end + 1)))); + } + + tempFile = createTempFileUnchecked(); + PresignedUrlDownloadRequest request = PresignedUrlDownloadRequest.builder() + .presignedUrl(presignedUrl) + .build(); + + s3AsyncClient.presignedUrlExtension() + .getObject(request, AsyncResponseTransformer.toFile(tempFile)) + .join(); + + assertThat(Files.readAllBytes(tempFile)).isEqualTo(data); + verify(13, getRequestedFor(urlEqualTo(PRESIGNED_URL_PATH))); + } + + @Test + void emptyObject_416OnFirstRequest_shouldFallbackToNonRangeGet() { + stubFor(get(urlEqualTo(PRESIGNED_URL_PATH)) + .withHeader("Range", matching("bytes=.*")) + .willReturn(aResponse() + .withStatus(416) + .withBody("InvalidRange" + + "The requested range is not satisfiable"))); + stubFor(get(urlEqualTo(PRESIGNED_URL_PATH)) + .withHeader("Range", absent()) + .willReturn(aResponse() + .withStatus(200) + .withHeader("Content-Length", "0") + .withHeader("ETag", "\"empty-etag\"") + .withBody(new byte[0]))); + + tempFile = createTempFileUnchecked(); + PresignedUrlDownloadRequest request = PresignedUrlDownloadRequest.builder() + .presignedUrl(presignedUrl) + .build(); + + s3AsyncClient.presignedUrlExtension() + .getObject(request, AsyncResponseTransformer.toFile(tempFile)) + .join(); + + assertThat(tempFile.toFile()).exists(); + assertThat(tempFile.toFile().length()).isEqualTo(0L); + } + + @Test + void multipartObject_416OnSecondRequest_shouldFailWithError() { + stubFor(get(urlEqualTo(PRESIGNED_URL_PATH)) + .inScenario("416-on-second") + .whenScenarioStateIs("Started") + .withHeader("Range", matching("bytes=0-15")) + .willReturn(aResponse() + .withStatus(206) + .withHeader("Content-Length", "16") + .withHeader("Content-Range", "bytes 0-15/32") + .withHeader("ETag", "\"test-etag\"") + .withBody(Arrays.copyOfRange(TEST_DATA, 0, 16))) + .willSetStateTo("first-done")); + + stubFor(get(urlEqualTo(PRESIGNED_URL_PATH)) + .inScenario("416-on-second") + .whenScenarioStateIs("first-done") + .willReturn(aResponse() + .withStatus(416) + .withBody("InvalidRange" + + "The requested range is not satisfiable"))); + + tempFile = createTempFileUnchecked(); + PresignedUrlDownloadRequest request = PresignedUrlDownloadRequest.builder() + .presignedUrl(presignedUrl) + .build(); + + assertThatThrownBy(() -> s3AsyncClient.presignedUrlExtension() + .getObject(request, AsyncResponseTransformer.toFile(tempFile)) + .join()) + .isInstanceOf(CompletionException.class); + } + + @Test + void getObject_firstPartContentLengthMismatch_shouldFailWithValidationError() throws IOException { + stubFor(get(urlEqualTo(PRESIGNED_URL_PATH)) + .withHeader("Range", matching("bytes=0-15")) + .willReturn(aResponse() + .withStatus(206) + .withHeader("Content-Length", "10") + .withHeader("Content-Range", "bytes 0-15/32") + .withHeader("ETag", "\"test-etag\"") + .withBody(Arrays.copyOfRange(TEST_DATA, 0, 10)))); + + tempFile = createTempFileUnchecked(); + PresignedUrlDownloadRequest request = PresignedUrlDownloadRequest.builder() + .presignedUrl(presignedUrl) + .build(); + + assertThatThrownBy(() -> s3AsyncClient.presignedUrlExtension() + .getObject(request, AsyncResponseTransformer.toFile(tempFile)) + .join()) + .isInstanceOf(CompletionException.class) + .hasRootCauseInstanceOf(SdkClientException.class) + .hasMessageContaining("content length validation failed"); + } + + @Test + void getObject_firstPartContentRangeStartByteMismatch_shouldFailWithValidationError() throws IOException { + stubFor(get(urlEqualTo(PRESIGNED_URL_PATH)) + .withHeader("Range", matching("bytes=0-15")) + .willReturn(aResponse() + .withStatus(206) + .withHeader("Content-Length", "16") + .withHeader("Content-Range", "bytes 5-20/32") + .withHeader("ETag", "\"test-etag\"") + .withBody(Arrays.copyOfRange(TEST_DATA, 0, 16)))); + + tempFile = createTempFileUnchecked(); + PresignedUrlDownloadRequest request = PresignedUrlDownloadRequest.builder() + .presignedUrl(presignedUrl) + .build(); + + assertThatThrownBy(() -> s3AsyncClient.presignedUrlExtension() + .getObject(request, AsyncResponseTransformer.toFile(tempFile)) + .join()) + .isInstanceOf(CompletionException.class) + .hasRootCauseInstanceOf(SdkClientException.class) + .hasMessageContaining("Content-Range mismatch"); + } + + @Test + void getObject_objectMutatedBetweenParts_shouldFailWith412() throws IOException { + stubFor(get(urlEqualTo(PRESIGNED_URL_PATH)) + .withHeader("Range", matching("bytes=0-15")) + .willReturn(aResponse() + .withStatus(206) + .withHeader("Content-Length", "16") + .withHeader("Content-Range", "bytes 0-15/32") + .withHeader("ETag", "\"original-etag\"") + .withBody(Arrays.copyOfRange(TEST_DATA, 0, 16)))); + + stubFor(get(urlEqualTo(PRESIGNED_URL_PATH)) + .withHeader("Range", matching("bytes=16-31")) + .withHeader("If-Match", matching("\"original-etag\"")) + .willReturn(aResponse() + .withStatus(412) + .withBody("PreconditionFailed" + + "Object changed"))); + + tempFile = createTempFileUnchecked(); + PresignedUrlDownloadRequest request = PresignedUrlDownloadRequest.builder() + .presignedUrl(presignedUrl) + .build(); + + assertThatThrownBy(() -> s3AsyncClient.presignedUrlExtension() + .getObject(request, AsyncResponseTransformer.toFile(tempFile)) + .join()) + .isInstanceOf(CompletionException.class); + } + + private static Path createTempFile() throws IOException { + Path path = Files.createTempFile("parallel-test-" + UUID.randomUUID(), ".tmp"); + Files.deleteIfExists(path); + return path; + } + + private static Path createTempFileUnchecked() { + try { + return createTempFile(); + } catch (IOException e) { + throw new RuntimeException(e); + } + } +} diff --git a/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/multipart/PresignedUrlDownloadHelperTest.java b/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/multipart/PresignedUrlDownloadHelperTest.java new file mode 100644 index 000000000000..7902916abf97 --- /dev/null +++ b/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/multipart/PresignedUrlDownloadHelperTest.java @@ -0,0 +1,182 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.services.s3.internal.multipart; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.net.MalformedURLException; +import java.net.URL; +import java.util.Optional; +import org.junit.jupiter.api.Test; +import software.amazon.awssdk.core.exception.SdkClientException; +import software.amazon.awssdk.services.s3.model.GetObjectResponse; +import software.amazon.awssdk.services.s3.presignedurl.model.PresignedUrlDownloadRequest; + +class PresignedUrlDownloadHelperTest { + + @Test + void validatePartResponse_validResponse_shouldReturnEmpty() { + GetObjectResponse response = GetObjectResponse.builder() + .contentRange("bytes 0-15/32") + .contentLength(16L) + .build(); + + Optional result = PresignedUrlDownloadHelper.validatePartResponse( + response, 0, 16L, 32L, 2L); + + assertThat(result).isEmpty(); + } + + @Test + void validatePartResponse_missingContentRange_shouldReturnError() { + GetObjectResponse response = GetObjectResponse.builder() + .contentLength(16L) + .build(); + + Optional result = PresignedUrlDownloadHelper.validatePartResponse( + response, 0, 16L, 32L, 2L); + + assertThat(result).isPresent(); + assertThat(result.get().getMessage()).contains("No Content-Range header"); + } + + @Test + void validatePartResponse_invalidContentLength_shouldReturnError() { + GetObjectResponse response = GetObjectResponse.builder() + .contentRange("bytes 0-15/32") + .contentLength(-1L) + .build(); + + Optional result = PresignedUrlDownloadHelper.validatePartResponse( + response, 0, 16L, 32L, 2L); + + assertThat(result).isPresent(); + assertThat(result.get().getMessage()).contains("Invalid or missing Content-Length"); + } + + @Test + void validatePartResponse_contentRangeMismatch_shouldReturnError() { + GetObjectResponse response = GetObjectResponse.builder() + .contentRange("bytes 5-20/32") + .contentLength(16L) + .build(); + + Optional result = PresignedUrlDownloadHelper.validatePartResponse( + response, 0, 16L, 32L, 2L); + + assertThat(result).isPresent(); + assertThat(result.get().getMessage()).contains("Content-Range mismatch for part 0"); + } + + @Test + void validatePartResponse_contentLengthMismatch_shouldReturnError() { + GetObjectResponse response = GetObjectResponse.builder() + .contentRange("bytes 0-15/32") + .contentLength(10L) + .build(); + + Optional result = PresignedUrlDownloadHelper.validatePartResponse( + response, 0, 16L, 32L, 2L); + + assertThat(result).isPresent(); + assertThat(result.get().getMessage()).contains("content length validation failed"); + } + + @Test + void validatePartResponse_lastPartSmallerSize_shouldPass() { + // 30-byte object, 16-byte parts → part 1 is bytes 16-29 (14 bytes) + GetObjectResponse response = GetObjectResponse.builder() + .contentRange("bytes 16-29/30") + .contentLength(14L) + .build(); + + Optional result = PresignedUrlDownloadHelper.validatePartResponse( + response, 1, 16L, 30L, 2L); + + assertThat(result).isEmpty(); + } + + @Test + void validatePartResponse_nullTotalContentLength_shouldStillValidateRange() { + GetObjectResponse response = GetObjectResponse.builder() + .contentRange("bytes 0-15/32") + .contentLength(16L) + .build(); + + // When totalContentLength is null, content-length check is skipped but range check still works + Optional result = PresignedUrlDownloadHelper.validatePartResponse( + response, 0, 16L, null, null); + + assertThat(result).isEmpty(); + } + + @Test + void createRangedGetRequest_firstPart_shouldNotIncludeIfMatch() throws MalformedURLException { + URL url = new URL("https://bucket.s3.amazonaws.com/key?X-Amz-Signature=abc"); + PresignedUrlDownloadRequest original = PresignedUrlDownloadRequest.builder() + .presignedUrl(url) + .build(); + + PresignedUrlDownloadRequest result = PresignedUrlDownloadHelper.createRangedGetRequest( + original, 0, 16L, 32L, "\"etag\""); + + assertThat(result.range()).isEqualTo("bytes=0-15"); + assertThat(result.ifMatch()).isNull(); + assertThat(result.presignedUrl()).isEqualTo(url); + } + + @Test + void createRangedGetRequest_secondPart_shouldIncludeIfMatch() throws MalformedURLException { + URL url = new URL("https://bucket.s3.amazonaws.com/key?X-Amz-Signature=abc"); + PresignedUrlDownloadRequest original = PresignedUrlDownloadRequest.builder() + .presignedUrl(url) + .build(); + + PresignedUrlDownloadRequest result = PresignedUrlDownloadHelper.createRangedGetRequest( + original, 1, 16L, 32L, "\"etag\""); + + assertThat(result.range()).isEqualTo("bytes=16-31"); + assertThat(result.ifMatch()).isEqualTo("\"etag\""); + } + + @Test + void createRangedGetRequest_lastPartClamped_shouldNotExceedTotalSize() throws MalformedURLException { + URL url = new URL("https://bucket.s3.amazonaws.com/key?X-Amz-Signature=abc"); + PresignedUrlDownloadRequest original = PresignedUrlDownloadRequest.builder() + .presignedUrl(url) + .build(); + + // 30-byte object, 16-byte parts → part 1 should be bytes=16-29 (not 16-31) + PresignedUrlDownloadRequest result = PresignedUrlDownloadHelper.createRangedGetRequest( + original, 1, 16L, 30L, "\"etag\""); + + assertThat(result.range()).isEqualTo("bytes=16-29"); + } + + @Test + void createRangedGetRequest_nullTotalContentLength_shouldUseFullPartSize() throws MalformedURLException { + URL url = new URL("https://bucket.s3.amazonaws.com/key?X-Amz-Signature=abc"); + PresignedUrlDownloadRequest original = PresignedUrlDownloadRequest.builder() + .presignedUrl(url) + .build(); + + // First part when total size unknown — uses full part size without clamping + PresignedUrlDownloadRequest result = PresignedUrlDownloadHelper.createRangedGetRequest( + original, 0, 16L, null, null); + + assertThat(result.range()).isEqualTo("bytes=0-15"); + } +} diff --git a/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/multipart/PresignedUrlMultipartDownloaderSubscriberRetryWiremockTest.java b/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/multipart/PresignedUrlMultipartDownloaderSubscriberRetryWiremockTest.java new file mode 100644 index 000000000000..fe81ea43715b --- /dev/null +++ b/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/multipart/PresignedUrlMultipartDownloaderSubscriberRetryWiremockTest.java @@ -0,0 +1,357 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.services.s3.internal.multipart; + +import static com.github.tomakehurst.wiremock.client.WireMock.aResponse; +import static com.github.tomakehurst.wiremock.client.WireMock.exactly; +import static com.github.tomakehurst.wiremock.client.WireMock.get; +import static com.github.tomakehurst.wiremock.client.WireMock.getRequestedFor; +import static com.github.tomakehurst.wiremock.client.WireMock.matching; +import static com.github.tomakehurst.wiremock.client.WireMock.moreThan; +import static com.github.tomakehurst.wiremock.client.WireMock.stubFor; +import static com.github.tomakehurst.wiremock.client.WireMock.urlEqualTo; +import static com.github.tomakehurst.wiremock.client.WireMock.verify; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.junit.jupiter.api.Assertions.assertArrayEquals; + +import com.github.tomakehurst.wiremock.http.Fault; +import com.github.tomakehurst.wiremock.junit5.WireMockRuntimeInfo; +import com.github.tomakehurst.wiremock.junit5.WireMockTest; +import com.github.tomakehurst.wiremock.stubbing.Scenario; +import java.io.IOException; +import java.net.MalformedURLException; +import java.net.URI; +import java.net.URL; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Arrays; +import java.util.UUID; +import java.util.concurrent.CompletionException; +import java.util.concurrent.TimeUnit; +import java.util.stream.Stream; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Timeout; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import software.amazon.awssdk.core.ResponseBytes; +import software.amazon.awssdk.core.async.AsyncResponseTransformer; +import software.amazon.awssdk.core.exception.SdkClientException; +import software.amazon.awssdk.services.s3.S3AsyncClient; +import software.amazon.awssdk.services.s3.model.GetObjectResponse; +import software.amazon.awssdk.services.s3.model.S3Exception; +import software.amazon.awssdk.services.s3.multipart.MultipartConfiguration; +import software.amazon.awssdk.services.s3.presignedurl.model.PresignedUrlDownloadRequest; + +@WireMockTest +@Timeout(value = 30, unit = TimeUnit.SECONDS) +class PresignedUrlMultipartDownloaderSubscriberRetryWiremockTest { + + private static final String PRESIGNED_URL_PATH = "/presigned-url"; + private static final int PART_SIZE = 16; + private static final byte[] TEST_DATA = "This is exactly a 32 byte string".getBytes(StandardCharsets.UTF_8); + private static final byte[] PART_1_DATA = Arrays.copyOfRange(TEST_DATA, 0, PART_SIZE); + private static final byte[] PART_2_DATA = Arrays.copyOfRange(TEST_DATA, PART_SIZE, TEST_DATA.length); + + private S3AsyncClient s3AsyncClient; + private URL presignedUrl; + private Path tempFile; + + @BeforeEach + void setup(WireMockRuntimeInfo wiremock) throws MalformedURLException { + s3AsyncClient = S3AsyncClient.builder() + .endpointOverride(URI.create(wiremock.getHttpBaseUrl())) + .multipartEnabled(true) + .multipartConfiguration(MultipartConfiguration.builder() + .minimumPartSizeInBytes((long) PART_SIZE) + .build()) + .build(); + presignedUrl = new URL(wiremock.getHttpBaseUrl() + PRESIGNED_URL_PATH); + } + + static Stream transformerTypes() { + return Stream.of( + Arguments.of("toBytes"), + Arguments.of("toFile") + ); + } + + @ParameterizedTest(name = "errorOnFirstPart_nonRetryable_shouldFailImmediately [{0}]") + @MethodSource("transformerTypes") + void errorOnFirstPart_nonRetryable_shouldFailImmediately(String transformerType) throws IOException { + stubError(1, 403, "AccessDeniedAccess denied"); + + assertThatThrownBy(() -> executeDownload(transformerType).join()) + .isInstanceOf(CompletionException.class) + .hasCauseInstanceOf(S3Exception.class) + .hasMessageContaining("Access denied"); + + verify(exactly(1), getRequestedFor(urlEqualTo(PRESIGNED_URL_PATH))); + } + + @ParameterizedTest(name = "errorOnMiddlePart_nonRetryable_shouldFail [{0}]") + @MethodSource("transformerTypes") + void errorOnMiddlePart_nonRetryable_shouldFail(String transformerType) throws IOException { + stubSuccessPart1(); + stubError(2, 403, "AccessDeniedAccess denied on part 2"); + + assertThatThrownBy(() -> executeDownload(transformerType).join()) + .isInstanceOf(CompletionException.class); + + verify(exactly(1), getRequestedFor(urlEqualTo(PRESIGNED_URL_PATH)) + .withHeader("Range", matching("bytes=0-15"))); + verify(exactly(1), getRequestedFor(urlEqualTo(PRESIGNED_URL_PATH)) + .withHeader("Range", matching("bytes=16-31"))); + } + + @ParameterizedTest(name = "errorOnFirstPart_retryable_exhaustsRetries_shouldFail [{0}]") + @MethodSource("transformerTypes") + void errorOnFirstPart_retryable_exhaustsRetries_shouldFail(String transformerType) throws IOException { + stubError(1, 500, "InternalErrorServer error"); + + assertThatThrownBy(() -> executeDownload(transformerType).join()) + .isInstanceOf(CompletionException.class) + .hasCauseInstanceOf(S3Exception.class) + .hasMessageContaining("Server error"); + + verify(moreThan(1), getRequestedFor(urlEqualTo(PRESIGNED_URL_PATH))); + } + + @ParameterizedTest(name = "errorOnMiddlePart_retryable_exhaustsRetries_shouldFail [{0}]") + @MethodSource("transformerTypes") + void errorOnMiddlePart_retryable_exhaustsRetries_shouldFail(String transformerType) throws IOException { + stubSuccessPart1(); + stubError(2, 500, "InternalErrorPermanent failure"); + + assertThatThrownBy(() -> executeDownload(transformerType).join()) + .isInstanceOf(CompletionException.class); + + verify(exactly(1), getRequestedFor(urlEqualTo(PRESIGNED_URL_PATH)) + .withHeader("Range", matching("bytes=0-15"))); + verify(moreThan(1), getRequestedFor(urlEqualTo(PRESIGNED_URL_PATH)) + .withHeader("Range", matching("bytes=16-31"))); + } + + @ParameterizedTest(name = "ioErrorOnFirstPart_exhaustsRetries_shouldFail [{0}]") + @MethodSource("transformerTypes") + void ioErrorOnFirstPart_exhaustsRetries_shouldFail(String transformerType) throws IOException { + stubIoError(1); + + assertThatThrownBy(() -> executeDownload(transformerType).join()) + .isInstanceOf(CompletionException.class) + .hasCauseInstanceOf(SdkClientException.class); + + verify(moreThan(1), getRequestedFor(urlEqualTo(PRESIGNED_URL_PATH)) + .withHeader("Range", matching("bytes=0-15"))); + } + + @ParameterizedTest(name = "errorOnFirstPart_retryable_thenSucceeds [{0}]") + @MethodSource("transformerTypes") + void errorOnFirstPart_retryable_thenSucceeds(String transformerType) throws IOException { + stubFor(get(urlEqualTo(PRESIGNED_URL_PATH)) + .withHeader("Range", matching("bytes=0-15")) + .inScenario("retry-first") + .whenScenarioStateIs(Scenario.STARTED) + .willReturn(aResponse() + .withStatus(500) + .withBody("InternalErrorretry")) + .willSetStateTo("retry")); + + stubFor(get(urlEqualTo(PRESIGNED_URL_PATH)) + .withHeader("Range", matching("bytes=0-15")) + .inScenario("retry-first") + .whenScenarioStateIs("retry") + .willReturn(successPart1Response()) + .willSetStateTo("part1-done")); + + stubFor(get(urlEqualTo(PRESIGNED_URL_PATH)) + .withHeader("Range", matching("bytes=16-31")) + .inScenario("retry-first") + .whenScenarioStateIs("part1-done") + .willReturn(successPart2Response())); + + Object result = executeDownload(transformerType).join(); + assertSuccessfulDownload(transformerType, result); + + verify(exactly(2), getRequestedFor(urlEqualTo(PRESIGNED_URL_PATH)) + .withHeader("Range", matching("bytes=0-15"))); + } + + @ParameterizedTest(name = "errorOnMiddlePart_retryable_thenSucceeds [{0}]") + @MethodSource("transformerTypes") + void errorOnMiddlePart_retryable_thenSucceeds(String transformerType) throws IOException { + stubSuccessPart1(); + + stubFor(get(urlEqualTo(PRESIGNED_URL_PATH)) + .withHeader("Range", matching("bytes=16-31")) + .inScenario("retry-middle") + .whenScenarioStateIs(Scenario.STARTED) + .willReturn(aResponse() + .withStatus(500) + .withBody("InternalErrorretry")) + .willSetStateTo("retry")); + + stubFor(get(urlEqualTo(PRESIGNED_URL_PATH)) + .withHeader("Range", matching("bytes=16-31")) + .inScenario("retry-middle") + .whenScenarioStateIs("retry") + .willReturn(successPart2Response())); + + Object result = executeDownload(transformerType).join(); + assertSuccessfulDownload(transformerType, result); + + verify(exactly(1), getRequestedFor(urlEqualTo(PRESIGNED_URL_PATH)) + .withHeader("Range", matching("bytes=0-15"))); + verify(exactly(2), getRequestedFor(urlEqualTo(PRESIGNED_URL_PATH)) + .withHeader("Range", matching("bytes=16-31"))); + } + + @ParameterizedTest(name = "ioErrorOnFirstPart_thenSucceeds [{0}]") + @MethodSource("transformerTypes") + void ioErrorOnFirstPart_thenSucceeds(String transformerType) throws IOException { + stubFor(get(urlEqualTo(PRESIGNED_URL_PATH)) + .withHeader("Range", matching("bytes=0-15")) + .inScenario("io-retry") + .whenScenarioStateIs(Scenario.STARTED) + .willReturn(aResponse().withFault(Fault.CONNECTION_RESET_BY_PEER)) + .willSetStateTo("retry")); + + stubFor(get(urlEqualTo(PRESIGNED_URL_PATH)) + .withHeader("Range", matching("bytes=0-15")) + .inScenario("io-retry") + .whenScenarioStateIs("retry") + .willReturn(successPart1Response()) + .willSetStateTo("part1-done")); + + stubFor(get(urlEqualTo(PRESIGNED_URL_PATH)) + .withHeader("Range", matching("bytes=16-31")) + .inScenario("io-retry") + .whenScenarioStateIs("part1-done") + .willReturn(successPart2Response())); + + Object result = executeDownload(transformerType).join(); + assertSuccessfulDownload(transformerType, result); + + verify(exactly(2), getRequestedFor(urlEqualTo(PRESIGNED_URL_PATH)) + .withHeader("Range", matching("bytes=0-15"))); + } + + @ParameterizedTest(name = "retryableError_thenUrlExpires_shouldFailWithExpiredError [{0}]") + @MethodSource("transformerTypes") + void retryableError_thenUrlExpires_shouldFailWithExpiredError(String transformerType) throws IOException { + stubFor(get(urlEqualTo(PRESIGNED_URL_PATH)) + .withHeader("Range", matching("bytes=0-15")) + .inScenario("url-expires") + .whenScenarioStateIs(Scenario.STARTED) + .willReturn(aResponse() + .withStatus(500) + .withBody("InternalErrorServer error")) + .willSetStateTo("expired")); + + stubFor(get(urlEqualTo(PRESIGNED_URL_PATH)) + .withHeader("Range", matching("bytes=0-15")) + .inScenario("url-expires") + .whenScenarioStateIs("expired") + .willReturn(aResponse() + .withStatus(403) + .withBody("AccessDenied" + + "Request has expired"))); + + assertThatThrownBy(() -> executeDownload(transformerType).join()) + .isInstanceOf(CompletionException.class) + .hasCauseInstanceOf(S3Exception.class) + .hasMessageContaining("Request has expired"); + + verify(exactly(2), getRequestedFor(urlEqualTo(PRESIGNED_URL_PATH))); + } + + + private java.util.concurrent.CompletableFuture executeDownload(String transformerType) throws IOException { + PresignedUrlDownloadRequest request = PresignedUrlDownloadRequest.builder() + .presignedUrl(presignedUrl) + .build(); + if ("toFile".equals(transformerType)) { + tempFile = Files.createTempFile("test-" + UUID.randomUUID(), ".tmp"); + Files.deleteIfExists(tempFile); + return s3AsyncClient.presignedUrlExtension().getObject(request, AsyncResponseTransformer.toFile(tempFile)); + } + return s3AsyncClient.presignedUrlExtension().getObject(request, AsyncResponseTransformer.toBytes()); + } + + @SuppressWarnings("unchecked") + private void assertSuccessfulDownload(String type, Object result) throws IOException { + if ("toBytes".equals(type)) { + assertArrayEquals(TEST_DATA, ((ResponseBytes) result).asByteArray()); + } else { + assertArrayEquals(TEST_DATA, Files.readAllBytes(tempFile)); + } + } + + private void stubSuccessPart1() { + stubFor(get(urlEqualTo(PRESIGNED_URL_PATH)) + .withHeader("Range", matching("bytes=0-15")) + .willReturn(successPart1Response())); + } + + private com.github.tomakehurst.wiremock.client.ResponseDefinitionBuilder successPart1Response() { + return aResponse() + .withStatus(206) + .withHeader("Content-Length", String.valueOf(PART_SIZE)) + .withHeader("Content-Range", "bytes 0-15/32") + .withHeader("ETag", "\"etag\"") + .withBody(PART_1_DATA); + } + + private com.github.tomakehurst.wiremock.client.ResponseDefinitionBuilder successPart2Response() { + return aResponse() + .withStatus(206) + .withHeader("Content-Length", String.valueOf(PART_SIZE)) + .withHeader("Content-Range", "bytes 16-31/32") + .withHeader("ETag", "\"etag\"") + .withBody(PART_2_DATA); + } + + private void stubError(int partNumber, int status, String body) { + String rangePattern = partNumber == 1 ? "bytes=0-15" : "bytes=16-31"; + stubFor(get(urlEqualTo(PRESIGNED_URL_PATH)) + .withHeader("Range", matching(rangePattern)) + .willReturn(aResponse() + .withStatus(status) + .withBody(body))); + } + + private void stubIoError(int partNumber) { + String rangePattern = partNumber == 1 ? "bytes=0-15" : "bytes=16-31"; + stubFor(get(urlEqualTo(PRESIGNED_URL_PATH)) + .withHeader("Range", matching(rangePattern)) + .willReturn(aResponse().withFault(Fault.CONNECTION_RESET_BY_PEER))); + } + + @AfterEach + void cleanup() { + if (s3AsyncClient != null) { + s3AsyncClient.close(); + } + if (tempFile != null && Files.exists(tempFile)) { + try { + Files.delete(tempFile); + } catch (IOException e) { + } + } + } +} diff --git a/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/multipart/PresignedUrlMultipartDownloaderSubscriberTckTest.java b/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/multipart/PresignedUrlMultipartDownloaderSubscriberTckTest.java new file mode 100644 index 000000000000..3c176d7e635a --- /dev/null +++ b/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/multipart/PresignedUrlMultipartDownloaderSubscriberTckTest.java @@ -0,0 +1,154 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.services.s3.internal.multipart; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.when; + +import java.net.MalformedURLException; +import java.nio.ByteBuffer; +import java.util.concurrent.CompletableFuture; +import org.mockito.Mockito; +import org.reactivestreams.Subscriber; +import org.reactivestreams.Subscription; +import org.reactivestreams.tck.SubscriberWhiteboxVerification; +import org.reactivestreams.tck.TestEnvironment; +import software.amazon.awssdk.core.async.AsyncResponseTransformer; +import software.amazon.awssdk.core.async.SdkPublisher; +import software.amazon.awssdk.services.s3.S3AsyncClient; +import software.amazon.awssdk.services.s3.model.GetObjectResponse; +import software.amazon.awssdk.services.s3.presignedurl.AsyncPresignedUrlExtension; +import software.amazon.awssdk.services.s3.presignedurl.model.PresignedUrlDownloadRequest; + +public class PresignedUrlMultipartDownloaderSubscriberTckTest + extends SubscriberWhiteboxVerification> { + + private S3AsyncClient s3mock; + + public PresignedUrlMultipartDownloaderSubscriberTckTest() { + super(new TestEnvironment()); + this.s3mock = Mockito.mock(S3AsyncClient.class); + } + + @Override + public Subscriber> + createSubscriber(WhiteboxSubscriberProbe> probe) { + AsyncPresignedUrlExtension presignedUrlExtension = Mockito.mock(AsyncPresignedUrlExtension.class); + when(s3mock.presignedUrlExtension()).thenReturn(presignedUrlExtension); + + CompletableFuture firstPartResponse = CompletableFuture.completedFuture( + GetObjectResponse.builder() + .contentRange("bytes 0-8388607/33554432") + .contentLength(8388608L) // 8MB + .eTag("\"test-etag-12345\"") + .build() + ); + + CompletableFuture subsequentPartResponse = CompletableFuture.completedFuture( + GetObjectResponse.builder() + .contentRange("bytes 8388608-16777215/33554432") + .contentLength(8388608L) // 8MB + .eTag("\"test-etag-12345\"") + .build() + ); + + when(presignedUrlExtension.getObject(any(PresignedUrlDownloadRequest.class), any(AsyncResponseTransformer.class))) + .thenReturn(firstPartResponse) + .thenReturn(subsequentPartResponse) + .thenReturn(subsequentPartResponse) + .thenReturn(subsequentPartResponse); + + return new PresignedUrlMultipartDownloaderSubscriber( + s3mock, + createTestPresignedUrlRequest(), + 8 * 1024 * 1024L, + new CompletableFuture<>() + ) { + @Override + public void onError(Throwable throwable) { + super.onError(throwable); + probe.registerOnError(throwable); + } + + @Override + public void onSubscribe(Subscription subscription) { + super.onSubscribe(subscription); + probe.registerOnSubscribe(new SubscriberPuppet() { + @Override + public void triggerRequest(long elements) { + subscription.request(elements); + } + + @Override + public void signalCancel() { + subscription.cancel(); + } + }); + } + + @Override + public void onNext(AsyncResponseTransformer item) { + super.onNext(item); + probe.registerOnNext(item); + } + + @Override + public void onComplete() { + super.onComplete(); + probe.registerOnComplete(); + } + }; + } + + @Override + public AsyncResponseTransformer createElement(int element) { + return new TestAsyncResponseTransformer(); + } + + private PresignedUrlDownloadRequest createTestPresignedUrlRequest() { + try { + return PresignedUrlDownloadRequest.builder() + .presignedUrl(java.net.URI.create("https://test-bucket.s3.amazonaws.com/test-key").toURL()) + .build(); + } catch (MalformedURLException e) { + throw new RuntimeException("Failed to create test URL", e); + } + } + + private static class TestAsyncResponseTransformer implements AsyncResponseTransformer { + private CompletableFuture future; + + @Override + public CompletableFuture prepare() { + this.future = new CompletableFuture<>(); + return this.future; + } + + @Override + public void onResponse(GetObjectResponse response) { + this.future.complete(response); + } + + @Override + public void onStream(SdkPublisher publisher) { + } + + @Override + public void exceptionOccurred(Throwable error) { + future.completeExceptionally(error); + } + } +} diff --git a/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/multipart/PresignedUrlMultipartDownloaderSubscriberWiremockTest.java b/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/multipart/PresignedUrlMultipartDownloaderSubscriberWiremockTest.java new file mode 100644 index 000000000000..ad05af389452 --- /dev/null +++ b/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/multipart/PresignedUrlMultipartDownloaderSubscriberWiremockTest.java @@ -0,0 +1,607 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.services.s3.internal.multipart; + +import static com.github.tomakehurst.wiremock.client.WireMock.aResponse; +import static com.github.tomakehurst.wiremock.client.WireMock.absent; +import static com.github.tomakehurst.wiremock.client.WireMock.get; +import static com.github.tomakehurst.wiremock.client.WireMock.matching; +import static com.github.tomakehurst.wiremock.client.WireMock.stubFor; +import static com.github.tomakehurst.wiremock.client.WireMock.urlEqualTo; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.junit.jupiter.api.Assertions.assertArrayEquals; + +import com.github.tomakehurst.wiremock.junit5.WireMockRuntimeInfo; +import com.github.tomakehurst.wiremock.junit5.WireMockTest; +import java.io.IOException; +import java.net.MalformedURLException; +import java.net.URI; +import java.net.URL; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Arrays; +import java.util.UUID; +import java.util.concurrent.CompletableFuture; +import java.util.stream.Stream; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import org.reactivestreams.Subscriber; +import org.reactivestreams.Subscription; +import software.amazon.awssdk.core.ResponseBytes; +import software.amazon.awssdk.core.async.AsyncResponseTransformer; +import software.amazon.awssdk.core.async.SdkPublisher; +import software.amazon.awssdk.core.exception.SdkClientException; +import software.amazon.awssdk.services.s3.S3AsyncClient; +import software.amazon.awssdk.services.s3.model.GetObjectResponse; +import software.amazon.awssdk.services.s3.model.S3Exception; +import software.amazon.awssdk.services.s3.multipart.MultipartConfiguration; +import software.amazon.awssdk.services.s3.presignedurl.model.PresignedUrlDownloadRequest; + +@WireMockTest +class PresignedUrlMultipartDownloaderSubscriberWiremockTest { + + private static final String PRESIGNED_URL_PATH = "/presigned-url"; + private static final byte[] TEST_DATA = "This is exactly a 32 byte string".getBytes(StandardCharsets.UTF_8); + + private S3AsyncClient s3AsyncClient; + private String presignedUrlBase; + private URL presignedUrl; + private Path tempFile; + + @BeforeEach + public void setup(WireMockRuntimeInfo wiremock) throws MalformedURLException { + MultipartConfiguration multipartConfig = MultipartConfiguration.builder() + .minimumPartSizeInBytes(16L) + .build(); + s3AsyncClient = S3AsyncClient.builder() + .endpointOverride(URI.create("http://localhost:" + wiremock.getHttpPort())) + .credentialsProvider(software.amazon.awssdk.auth.credentials.StaticCredentialsProvider.create( + software.amazon.awssdk.auth.credentials.AwsBasicCredentials.create("accessKey", "secretKey"))) + .region(software.amazon.awssdk.regions.Region.US_EAST_1) + .multipartEnabled(true) + .multipartConfiguration(multipartConfig) + .build(); + presignedUrlBase = "http://localhost:" + wiremock.getHttpPort(); + presignedUrl = createPresignedUrl(); + } + + static Stream transformerTypes() { + return Stream.of( + Arguments.of("toBytes"), + Arguments.of("toFile") + ); + } + + private CompletableFuture executeDownload(PresignedUrlDownloadRequest request, String transformerType) + throws IOException { + if ("toFile".equals(transformerType)) { + tempFile = createUniqueTempFile(); + return s3AsyncClient.presignedUrlExtension().getObject(request, AsyncResponseTransformer.toFile(tempFile)); + } + return s3AsyncClient.presignedUrlExtension().getObject(request, AsyncResponseTransformer.toBytes()); + } + + @SuppressWarnings("unchecked") + private void assertSuccessfulDownload(String type, Object result) throws IOException { + if ("toBytes".equals(type)) { + assertArrayEquals(TEST_DATA, ((ResponseBytes) result).asByteArray()); + } else { + assertThat(tempFile.toFile()).exists(); + byte[] fileContent = Files.readAllBytes(tempFile); + assertArrayEquals(TEST_DATA, fileContent); + } + } + + @ParameterizedTest(name = "presignedUrlDownload_withMultipartData_shouldReceiveCompleteBody [{0}]") + @MethodSource("transformerTypes") + void presignedUrlDownload_withMultipartData_shouldReceiveCompleteBody(String transformerType) throws IOException { + stubSuccessfulPresignedUrlResponse(); + PresignedUrlDownloadRequest request = PresignedUrlDownloadRequest.builder() + .presignedUrl(presignedUrl) + .build(); + Object result = executeDownload(request, transformerType).join(); + assertSuccessfulDownload(transformerType, result); + } + + @ParameterizedTest(name = "presignedUrlDownload_smallObjectSmallerThanPartSize_shouldSucceed [{0}]") + @MethodSource("transformerTypes") + @SuppressWarnings("unchecked") + void presignedUrlDownload_smallObjectSmallerThanPartSize_shouldSucceed(String transformerType) throws IOException { + byte[] smallData = "0123456789".getBytes(StandardCharsets.UTF_8); + stubFor(get(urlEqualTo(PRESIGNED_URL_PATH)) + .willReturn(aResponse() + .withStatus(206) + .withHeader("Content-Type", "application/octet-stream") + .withHeader("Content-Length", "10") + .withHeader("Content-Range", "bytes 0-9/10") + .withHeader("ETag", "\"small-etag\"") + .withBody(smallData))); + + PresignedUrlDownloadRequest request = PresignedUrlDownloadRequest.builder() + .presignedUrl(presignedUrl) + .build(); + Object result = executeDownload(request, transformerType).join(); + if ("toBytes".equals(transformerType)) { + assertArrayEquals(smallData, ((ResponseBytes) result).asByteArray()); + } else { + assertThat(tempFile.toFile()).exists(); + assertArrayEquals(smallData, Files.readAllBytes(tempFile)); + } + } + + @ParameterizedTest(name = "presignedUrlDownload_withRangeHeader_shouldReceivePartialContent [{0}]") + @MethodSource("transformerTypes") + @SuppressWarnings("unchecked") + void presignedUrlDownload_withRangeHeader_shouldReceivePartialContent(String transformerType) throws IOException { + stubSuccessfulRangeResponse(); + PresignedUrlDownloadRequest request = PresignedUrlDownloadRequest.builder() + .presignedUrl(presignedUrl) + .range("bytes=0-10") + .build(); + Object result = executeDownload(request, transformerType).join(); + byte[] expectedPartial = Arrays.copyOfRange(TEST_DATA, 0, 11); + if ("toBytes".equals(transformerType)) { + assertArrayEquals(expectedPartial, ((ResponseBytes) result).asByteArray()); + } else { + byte[] fileContent = Files.readAllBytes(tempFile); + assertArrayEquals(expectedPartial, fileContent); + } + } + + @ParameterizedTest(name = "presignedUrlDownload_whenRequestFails_shouldThrowException [{0}]") + @MethodSource("transformerTypes") + void presignedUrlDownload_whenRequestFails_shouldThrowException(String transformerType) { + stubFailedPresignedUrlResponse(); + PresignedUrlDownloadRequest request = PresignedUrlDownloadRequest.builder() + .presignedUrl(presignedUrl) + .build(); + assertThatThrownBy(() -> executeDownload(request, transformerType).join()) + .hasRootCauseInstanceOf(S3Exception.class); + } + + @ParameterizedTest(name = "presignedUrlDownload_whenFirstRequestFails_shouldThrowException [{0}]") + @MethodSource("transformerTypes") + void presignedUrlDownload_whenFirstRequestFails_shouldThrowException(String transformerType) { + stubInternalServerError(); + PresignedUrlDownloadRequest request = PresignedUrlDownloadRequest.builder() + .presignedUrl(presignedUrl) + .build(); + assertThatThrownBy(() -> executeDownload(request, transformerType).join()) + .hasRootCauseInstanceOf(S3Exception.class); + } + + @ParameterizedTest(name = "presignedUrlDownload_whenSecondRequestFails_shouldThrowException [{0}]") + @MethodSource("transformerTypes") + void presignedUrlDownload_whenSecondRequestFails_shouldThrowException(String transformerType) { + stubPartialFailureScenario(); + PresignedUrlDownloadRequest request = PresignedUrlDownloadRequest.builder() + .presignedUrl(presignedUrl) + .build(); + assertThatThrownBy(() -> executeDownload(request, transformerType).join()) + .hasRootCauseInstanceOf(S3Exception.class); + } + + @ParameterizedTest(name = "presignedUrlDownload_whenIOErrorOccurs_shouldThrowException [{0}]") + @MethodSource("transformerTypes") + void presignedUrlDownload_whenIOErrorOccurs_shouldThrowException(String transformerType) { + stubConnectionReset(); + PresignedUrlDownloadRequest request = PresignedUrlDownloadRequest.builder() + .presignedUrl(presignedUrl) + .build(); + assertThatThrownBy(() -> executeDownload(request, transformerType).join()) + .hasCauseInstanceOf(SdkClientException.class); + } + + @ParameterizedTest(name = "presignedUrlDownload_withMissingContentRange_shouldFailRequest [{0}]") + @MethodSource("transformerTypes") + void presignedUrlDownload_withMissingContentRange_shouldFailRequest(String transformerType) { + stubResponseWithMissingContentRange(); + PresignedUrlDownloadRequest request = PresignedUrlDownloadRequest.builder() + .presignedUrl(presignedUrl) + .build(); + assertThatThrownBy(() -> executeDownload(request, transformerType).join()) + .hasRootCauseInstanceOf(SdkClientException.class) + .hasMessageContaining("No Content-Range header in response"); + } + + @ParameterizedTest(name = "presignedUrlDownload_withInvalidContentLength_shouldFailRequest [{0}]") + @MethodSource("transformerTypes") + void presignedUrlDownload_withInvalidContentLength_shouldFailRequest(String transformerType) { + stubResponseWithInvalidContentLength(); + PresignedUrlDownloadRequest request = PresignedUrlDownloadRequest.builder() + .presignedUrl(presignedUrl) + .build(); + assertThatThrownBy(() -> executeDownload(request, transformerType).join()) + .hasRootCauseInstanceOf(SdkClientException.class) + .hasMessageContaining("Invalid or missing Content-Length in response"); + } + + @ParameterizedTest(name = "presignedUrlDownload_withContentRangeMismatch_shouldFailRequest [{0}]") + @MethodSource("transformerTypes") + void presignedUrlDownload_withContentRangeMismatch_shouldFailRequest(String transformerType) { + stubResponseWithContentRangeMismatch(); + PresignedUrlDownloadRequest request = PresignedUrlDownloadRequest.builder() + .presignedUrl(presignedUrl) + .build(); + assertThatThrownBy(() -> executeDownload(request, transformerType).join()) + .hasRootCauseInstanceOf(SdkClientException.class) + .hasMessageContaining("Content-Range mismatch"); + } + + @ParameterizedTest(name = "presignedUrlDownload_withContentLengthMismatch_shouldFailRequest [{0}]") + @MethodSource("transformerTypes") + void presignedUrlDownload_withContentLengthMismatch_shouldFailRequest(String transformerType) { + stubResponseWithContentLengthMismatch(); + PresignedUrlDownloadRequest request = PresignedUrlDownloadRequest.builder() + .presignedUrl(presignedUrl) + .build(); + assertThatThrownBy(() -> executeDownload(request, transformerType).join()) + .hasRootCauseInstanceOf(SdkClientException.class); + } + + @Test + void onNext_withNullTransformer_shouldThrowException() { + PresignedUrlMultipartDownloaderSubscriber subscriber = createTestSubscriber(); + + assertThatThrownBy(() -> subscriber.onNext(null)) + .isInstanceOf(NullPointerException.class) + .hasMessageContaining("onNext must not be called with null asyncResponseTransformer"); + } + + @ParameterizedTest(name = "presignedUrlDownload_emptyObject_shouldFallbackToNonRangeGet [{0}]") + @MethodSource("transformerTypes") + @SuppressWarnings("unchecked") + void presignedUrlDownload_emptyObject_shouldFallbackToNonRangeGet(String transformerType) throws IOException { + stubFor(get(urlEqualTo(PRESIGNED_URL_PATH)) + .withHeader("Range", matching("bytes=.*")) + .willReturn(aResponse() + .withStatus(416) + .withBody("InvalidRange" + + "The requested range is not satisfiable"))); + stubFor(get(urlEqualTo(PRESIGNED_URL_PATH)) + .withHeader("Range", absent()) + .willReturn(aResponse() + .withStatus(200) + .withHeader("Content-Length", "0") + .withHeader("ETag", "\"empty-etag\"") + .withBody(new byte[0]))); + + PresignedUrlDownloadRequest request = PresignedUrlDownloadRequest.builder() + .presignedUrl(presignedUrl) + .build(); + Object result = executeDownload(request, transformerType).join(); + if ("toBytes".equals(transformerType)) { + ResponseBytes bytes = (ResponseBytes) result; + assertThat(bytes.asByteArray()).isEmpty(); + } else { + assertThat(tempFile.toFile()).exists(); + assertThat(Files.size(tempFile)).isEqualTo(0L); + } + } + + @ParameterizedTest(name = "presignedUrlDownload_416OnSecondRequest_shouldFailWithError [{0}]") + @MethodSource("transformerTypes") + void presignedUrlDownload_416OnSecondRequest_shouldFailWithError(String transformerType) { + stubFor(get(urlEqualTo(PRESIGNED_URL_PATH)) + .inScenario("416-on-second") + .whenScenarioStateIs("Started") + .withHeader("Range", matching("bytes=0-15")) + .willReturn(aResponse() + .withStatus(206) + .withHeader("Content-Length", "16") + .withHeader("Content-Range", "bytes 0-15/32") + .withHeader("ETag", "\"test-etag\"") + .withBody(Arrays.copyOfRange(TEST_DATA, 0, 16))) + .willSetStateTo("first-done")); + + stubFor(get(urlEqualTo(PRESIGNED_URL_PATH)) + .inScenario("416-on-second") + .whenScenarioStateIs("first-done") + .willReturn(aResponse() + .withStatus(416) + .withBody("InvalidRange" + + "The requested range is not satisfiable"))); + + PresignedUrlDownloadRequest request = PresignedUrlDownloadRequest.builder() + .presignedUrl(presignedUrl) + .build(); + assertThatThrownBy(() -> executeDownload(request, transformerType).join()) + .hasRootCauseInstanceOf(S3Exception.class); + } + + @ParameterizedTest(name = "presignedUrlDownload_withRangeHeader_emptyObject_shouldThrow416 [{0}]") + @MethodSource("transformerTypes") + void presignedUrlDownload_withRangeHeader_emptyObject_shouldThrow416(String transformerType) { + stubFor(get(urlEqualTo(PRESIGNED_URL_PATH)) + .willReturn(aResponse() + .withStatus(416) + .withBody("InvalidRange" + + "The requested range is not satisfiable"))); + + PresignedUrlDownloadRequest request = PresignedUrlDownloadRequest.builder() + .presignedUrl(presignedUrl) + .range("bytes=0-1024") + .build(); + assertThatThrownBy(() -> executeDownload(request, transformerType).join()) + .hasRootCauseInstanceOf(S3Exception.class); + } + + /** + * Verifies that custom serial transformers on the presigned URL path correctly trigger + * the empty-object 416 fallback. + */ + @Test + void presignedUrlDownload_emptyObject_customTransformer_fallbackWorks() { + // Range request → 416 (simulates empty object race) + stubFor(get(urlEqualTo(PRESIGNED_URL_PATH)) + .withHeader("Range", matching("bytes=.*")) + .willReturn(aResponse() + .withStatus(416) + .withBody("InvalidRange" + + "The requested range is not satisfiable"))); + + // Non-range fallback GET → 200 with empty body (the correct fallback for empty object) + stubFor(get(urlEqualTo(PRESIGNED_URL_PATH)) + .withHeader("Range", absent()) + .willReturn(aResponse() + .withStatus(200) + .withHeader("Content-Length", "0") + .withBody(""))); + + AsyncResponseTransformer customTransformer = + new AsyncResponseTransformer() { + private CompletableFuture future; + @Override + public CompletableFuture prepare() { + future = new CompletableFuture<>(); + return future; + } + @Override public void onResponse(GetObjectResponse r) { } + @Override public void onStream(SdkPublisher p) { + p.subscribe(new Subscriber() { + @Override public void onSubscribe(Subscription s) { s.request(Long.MAX_VALUE); } + @Override public void onNext(ByteBuffer b) { } + @Override public void onError(Throwable t) { future.completeExceptionally(t); } + @Override public void onComplete() { future.complete("done"); } + }); + } + @Override public void exceptionOccurred(Throwable e) { future.completeExceptionally(e); } + }; + + PresignedUrlDownloadRequest request = PresignedUrlDownloadRequest.builder() + .presignedUrl(presignedUrl) + .build(); + + // Should succeed: 416 triggers fallback → non-range GET returns 200 + String result = s3AsyncClient.presignedUrlExtension() + .getObject(request, customTransformer) + .join(); + assertThat(result).isEqualTo("done"); + } + + @AfterEach + void cleanup() { + if (tempFile != null && Files.exists(tempFile)) { + try { + Files.delete(tempFile); + } catch (IOException e) { + } + } + } + + private static Path createUniqueTempFile() throws IOException { + String uniqueName = "test-" + UUID.randomUUID().toString(); + Path tempFile = Files.createTempFile(uniqueName, ".tmp"); + Files.deleteIfExists(tempFile); + return tempFile; + } + + private void stubSuccessfulPresignedUrlResponse() { + // Stub for first part (bytes 0-15) + stubFor(get(urlEqualTo(PRESIGNED_URL_PATH)) + .withHeader("Range", matching("bytes=0-15")) + .willReturn(aResponse() + .withStatus(206) + .withHeader("Content-Type", "application/octet-stream") + .withHeader("Content-Length", "16") + .withHeader("Content-Range", "bytes 0-15/32") + .withHeader("ETag", "\"test-etag\"") + .withBody(Arrays.copyOfRange(TEST_DATA, 0, 16)))); + + // Stub for second part (bytes 16-31) + stubFor(get(urlEqualTo(PRESIGNED_URL_PATH)) + .withHeader("Range", matching("bytes=16-31")) + .willReturn(aResponse() + .withStatus(206) + .withHeader("Content-Type", "application/octet-stream") + .withHeader("Content-Length", "16") + .withHeader("Content-Range", "bytes 16-31/32") + .withHeader("ETag", "\"test-etag\"") + .withBody(Arrays.copyOfRange(TEST_DATA, 16, 32)))); + } + + private void stubSuccessfulRangeResponse() { + byte[] partialData = Arrays.copyOfRange(TEST_DATA, 0, 11); + stubFor(get(urlEqualTo(PRESIGNED_URL_PATH)) + .willReturn(aResponse() + .withStatus(206) + .withHeader("Content-Type", "application/octet-stream") + .withHeader("Content-Length", String.valueOf(partialData.length)) + .withHeader("Content-Range", "bytes 0-10/" + TEST_DATA.length) + .withHeader("ETag", "\"test-etag\"") + .withBody(partialData))); + } + + private void stubFailedPresignedUrlResponse() { + stubFor(get(urlEqualTo(PRESIGNED_URL_PATH)) + .willReturn(aResponse() + .withStatus(404) + .withBody("NoSuchKeyThe specified key does not exist."))); + } + + private void stubInternalServerError() { + stubFor(get(urlEqualTo(PRESIGNED_URL_PATH)) + .willReturn(aResponse() + .withStatus(500) + .withBody("InternalErrorInternal Server Error"))); + } + + private void stubPartialFailureScenario() { + stubFor(get(urlEqualTo(PRESIGNED_URL_PATH)) + .inScenario("partial-failure") + .whenScenarioStateIs("Started") + .willReturn(aResponse() + .withStatus(206) + .withHeader("Content-Type", "application/octet-stream") + .withHeader("Content-Length", "16") + .withHeader("Content-Range", "bytes 0-15/" + TEST_DATA.length) + .withHeader("ETag", "\"test-etag\"") + .withBody(Arrays.copyOfRange(TEST_DATA, 0, 16))) + .willSetStateTo("first-success")); + + stubFor(get(urlEqualTo(PRESIGNED_URL_PATH)) + .inScenario("partial-failure") + .whenScenarioStateIs("first-success") + .willReturn(aResponse() + .withStatus(500) + .withBody("InternalErrorSecond request failed"))); + } + + private void stubConnectionReset() { + stubFor(get(urlEqualTo(PRESIGNED_URL_PATH)) + .willReturn(aResponse() + .withFault(com.github.tomakehurst.wiremock.http.Fault.CONNECTION_RESET_BY_PEER))); + } + + private URL createPresignedUrl() throws MalformedURLException { + return new URL(presignedUrlBase + PRESIGNED_URL_PATH); + } + + private PresignedUrlMultipartDownloaderSubscriber createTestSubscriber() { + return new PresignedUrlMultipartDownloaderSubscriber( + s3AsyncClient, + PresignedUrlDownloadRequest.builder().presignedUrl(presignedUrl).build(), + 1024L, + new CompletableFuture<>()); + } + + private void stubResponseWithMissingContentRange() { + stubFor(get(urlEqualTo(PRESIGNED_URL_PATH)) + .willReturn(aResponse() + .withStatus(206) + .withHeader("Content-Type", "application/octet-stream") + .withHeader("Content-Length", "16") + .withHeader("ETag", "\"test-etag\"") + .withBody(Arrays.copyOfRange(TEST_DATA, 0, 16)))); + } + + private void stubResponseWithInvalidContentLength() { + stubFor(get(urlEqualTo(PRESIGNED_URL_PATH)) + .willReturn(aResponse() + .withStatus(206) + .withHeader("Content-Type", "application/octet-stream") + .withHeader("Content-Range", "bytes 0-15/" + TEST_DATA.length) + .withHeader("ETag", "\"test-etag\"") + .withBody(Arrays.copyOfRange(TEST_DATA, 0, 16)))); + } + + private void stubResponseWithContentRangeMismatch() { + stubFor(get(urlEqualTo(PRESIGNED_URL_PATH)) + .willReturn(aResponse() + .withStatus(206) + .withHeader("Content-Type", "application/octet-stream") + .withHeader("Content-Length", "16") + .withHeader("Content-Range", "bytes 5000-5015/" + TEST_DATA.length) + .withHeader("ETag", "\"test-etag\"") + .withBody(Arrays.copyOfRange(TEST_DATA, 0, 16)))); + } + + private void stubResponseWithContentLengthMismatch() { + stubFor(get(urlEqualTo(PRESIGNED_URL_PATH)) + .willReturn(aResponse() + .withStatus(206) + .withHeader("Content-Type", "application/octet-stream") + .withHeader("Content-Length", "8") + .withHeader("Content-Range", "bytes 0-15/" + TEST_DATA.length) + .withHeader("ETag", "\"test-etag\"") + .withBody(Arrays.copyOfRange(TEST_DATA, 0, 8)))); + } + + @Test + void presignedUrlDownload_customTransformer_hasFullObjectMetadata() { + // Stub two parts: 16 bytes each, total 32 + stubFor(get(urlEqualTo(PRESIGNED_URL_PATH)) + .withHeader("Range", matching("bytes=0-15")) + .willReturn(aResponse() + .withStatus(206) + .withHeader("Content-Length", "16") + .withHeader("Content-Range", "bytes 0-15/32") + .withHeader("ETag", "\"test-etag\"") + .withBody(Arrays.copyOfRange(TEST_DATA, 0, 16)))); + + stubFor(get(urlEqualTo(PRESIGNED_URL_PATH)) + .withHeader("Range", matching("bytes=16-31")) + .willReturn(aResponse() + .withStatus(206) + .withHeader("Content-Length", "16") + .withHeader("Content-Range", "bytes 16-31/32") + .withHeader("ETag", "\"test-etag\"") + .withBody(Arrays.copyOfRange(TEST_DATA, 16, 32)))); + + AsyncResponseTransformer customTransformer = + new AsyncResponseTransformer() { + private CompletableFuture future; + private GetObjectResponse response; + + @Override + public CompletableFuture prepare() { + future = new CompletableFuture<>(); + return future; + } + @Override public void onResponse(GetObjectResponse r) { this.response = r; } + @Override public void onStream(SdkPublisher p) { + p.subscribe(new Subscriber() { + @Override public void onSubscribe(Subscription s) { s.request(Long.MAX_VALUE); } + @Override public void onNext(ByteBuffer b) { } + @Override public void onError(Throwable t) { future.completeExceptionally(t); } + @Override public void onComplete() { + future.complete("contentLength=" + response.contentLength() + + "|contentRange=" + response.contentRange()); + } + }); + } + @Override public void exceptionOccurred(Throwable e) { future.completeExceptionally(e); } + }; + + PresignedUrlDownloadRequest request = PresignedUrlDownloadRequest.builder() + .presignedUrl(presignedUrl) + .build(); + + String result = s3AsyncClient.presignedUrlExtension() + .getObject(request, customTransformer) + .join(); + + assertThat(result).contains("contentLength=32"); + assertThat(result).contains("contentRange=bytes 0-31/32"); + } +} diff --git a/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/presignedurl/DefaultAsyncPresignedUrlExtensionTest.java b/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/presignedurl/DefaultAsyncPresignedUrlExtensionTest.java new file mode 100644 index 000000000000..9fed4335aa5a --- /dev/null +++ b/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/presignedurl/DefaultAsyncPresignedUrlExtensionTest.java @@ -0,0 +1,438 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.services.s3.internal.presignedurl; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.Mockito.any; +import static org.mockito.Mockito.atLeastOnce; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; + +import java.io.ByteArrayInputStream; +import java.net.URI; +import java.net.URL; +import java.nio.charset.StandardCharsets; +import java.util.Collections; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Executors; +import java.util.function.Consumer; +import java.util.stream.Stream; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import org.mockito.ArgumentCaptor; +import software.amazon.awssdk.auth.credentials.AnonymousCredentialsProvider; +import software.amazon.awssdk.awscore.client.config.AwsAdvancedClientOption; +import software.amazon.awssdk.awscore.client.config.AwsClientOption; +import software.amazon.awssdk.awscore.client.handler.AwsAsyncClientHandler; +import software.amazon.awssdk.awscore.internal.AwsProtocolMetadata; +import software.amazon.awssdk.awscore.internal.AwsServiceProtocol; +import software.amazon.awssdk.awscore.retry.AwsRetryStrategy; +import software.amazon.awssdk.core.ResponseBytes; +import software.amazon.awssdk.core.async.AsyncResponseTransformer; +import software.amazon.awssdk.core.client.config.SdkAdvancedAsyncClientOption; +import software.amazon.awssdk.core.client.config.SdkAdvancedClientOption; +import software.amazon.awssdk.core.client.config.SdkClientConfiguration; +import software.amazon.awssdk.core.client.config.SdkClientOption; +import software.amazon.awssdk.core.client.handler.AsyncClientHandler; +import software.amazon.awssdk.core.exception.SdkClientException; +import software.amazon.awssdk.core.metrics.CoreMetric; +import software.amazon.awssdk.http.AbortableInputStream; +import software.amazon.awssdk.http.HttpExecuteResponse; +import software.amazon.awssdk.http.SdkHttpFullResponse; +import software.amazon.awssdk.http.SdkHttpMethod; +import software.amazon.awssdk.http.SdkHttpRequest; +import software.amazon.awssdk.metrics.MetricCollection; +import software.amazon.awssdk.metrics.MetricPublisher; +import software.amazon.awssdk.protocols.core.ExceptionMetadata; +import software.amazon.awssdk.protocols.xml.AwsS3ProtocolFactory; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.s3.model.GetObjectResponse; +import software.amazon.awssdk.services.s3.model.InvalidObjectStateException; +import software.amazon.awssdk.services.s3.model.NoSuchKeyException; +import software.amazon.awssdk.services.s3.model.S3Exception; +import software.amazon.awssdk.services.s3.presignedurl.model.PresignedUrlDownloadRequest; +import software.amazon.awssdk.testutils.service.http.MockAsyncHttpClient; + +/** + * Tests for {@link DefaultAsyncPresignedUrlExtension} using MockAsyncHttpClient to verify HTTP interactions. + */ +class DefaultAsyncPresignedUrlExtensionTest { + + private static final String TEST_CONTENT = "test-content"; + private static final URI DEFAULT_ENDPOINT = URI.create("https://defaultendpoint.com"); + private static final String TEST_URL = "https://test-bucket.s3.us-east-1.amazonaws.com/test-key?" + + "X-Amz-Date=20250707T000000Z&" + + "X-Amz-Signature=test-signature-value&" + + "X-Amz-Algorithm=AWS4-HMAC-SHA256&" + + "X-Amz-SignedHeaders=host&" + + "X-Amz-Security-Token=test-session-token&" + + "X-Amz-Credential=AKIAIOSFODNN7EXAMPLE%2F20250707%2Fus-east-1%2Fs3%2Faws4_request&" + + "X-Amz-Expires=86400"; + + private MockAsyncHttpClient mockHttpClient; + private DefaultAsyncPresignedUrlExtension presignedUrlExtension; + private PresignedUrlDownloadRequest testRequest; + private AwsProtocolMetadata protocolMetadata; + private AwsS3ProtocolFactory protocolFactory; + private AsyncClientHandler clientHandler; + + @BeforeEach + void setUp() throws Exception { + mockHttpClient = new MockAsyncHttpClient(); + URL testPresignedUrl = new URL(TEST_URL); + testRequest = PresignedUrlDownloadRequest.builder() + .presignedUrl(testPresignedUrl) + .build(); + + SdkClientConfiguration clientConfiguration = getDefaultSdkConfigs(); + protocolMetadata = AwsProtocolMetadata.builder() + .serviceProtocol(AwsServiceProtocol.REST_XML) + .build(); + protocolFactory = initProtocolFactory(clientConfiguration); + clientHandler = new AwsAsyncClientHandler(clientConfiguration); + + presignedUrlExtension = new DefaultAsyncPresignedUrlExtension( + clientHandler, protocolFactory, clientConfiguration, protocolMetadata); + } + + private static Stream httpResponseTestCases() { + return Stream.of( + Arguments.of( + "Success response", + createSuccessResponse(), + true, + null + ), + Arguments.of( + "404 Not Found response", + createErrorResponse(404, "NoSuchKey", "The specified key does not exist."), + false, + NoSuchKeyException.class + ), + Arguments.of( + "403 Invalid Object State response", + createErrorResponse(403, "InvalidObjectState", "The operation is not valid for the object's storage class."), + false, + InvalidObjectStateException.class + ), + Arguments.of( + "Generic error response", + createErrorResponse(500, "InternalError", "We encountered an internal error. Please try again."), + false, + S3Exception.class + ) + ); + } + + private static Stream requestConfigurationTestCases() { + return Stream.of( + Arguments.of( + "Basic request", + (Consumer) builder -> + builder.presignedUrl(createTestUrl()), + null + ), + Arguments.of( + "Request with range header", + (Consumer) builder -> + builder.presignedUrl(createTestUrl()).range("bytes=0-1024"), + "bytes=0-1024" + ) + ); + } + + private static Stream additionalTestCases() { + return Stream.of( + Arguments.of("Custom transformer test", "CUSTOM_TRANSFORMER"), + Arguments.of("Metrics collection test", "METRICS_COLLECTION") + ); + } + + private static Stream invalidUrlTestCases() { + return Stream.of( + Arguments.of("Invalid URL format", "not-a-url"), + Arguments.of("Empty HTTP URL", "http://"), + Arguments.of("Empty HTTPS URL", "https://"), + Arguments.of("Empty string", "") + ); + } + + @ParameterizedTest(name = "{0}") + @MethodSource("httpResponseTestCases") + void given_AsyncPresignedUrlExtension_when_GetObjectWithDifferentHttpResponses_then_ShouldHandleSuccessAndErrorsCorrectly( + String testName, + HttpExecuteResponse response, + boolean expectSuccess, + Class expectedExceptionType) { + + mockHttpClient.stubNextResponse(response); + + if (expectSuccess) { + assertSuccessfulGetObject(testRequest); + } else { + CompletableFuture> future = + presignedUrlExtension.getObject(testRequest, AsyncResponseTransformer.toBytes()); + + assertThatThrownBy(future::join) + .hasCauseInstanceOf(expectedExceptionType); + } + } + + @ParameterizedTest(name = "{0}") + @MethodSource("requestConfigurationTestCases") + void given_AsyncPresignedUrlExtension_when_GetObjectWithDifferentRequestConfigurations_then_ShouldSetCorrectHeaders( + String testName, + Consumer requestCustomizer, + String expectedRangeHeader) throws ExecutionException, InterruptedException { + + mockHttpClient.stubNextResponse(createSuccessResponse()); + + PresignedUrlDownloadRequest.Builder builder = PresignedUrlDownloadRequest.builder(); + requestCustomizer.accept(builder); + PresignedUrlDownloadRequest request = builder.build(); + + CompletableFuture> future = + presignedUrlExtension.getObject(request, AsyncResponseTransformer.toBytes()); + ResponseBytes result = future.get(); + + assertThat(result).isNotNull(); + assertThat(result.asUtf8String()).isEqualTo(TEST_CONTENT); + + SdkHttpRequest lastRequest = mockHttpClient.getLastRequest(); + assertThat(lastRequest.method()).isEqualTo(SdkHttpMethod.GET); + + if (expectedRangeHeader != null) { + assertThat(lastRequest.firstMatchingHeader("Range")) + .isPresent() + .contains(expectedRangeHeader); + } + } + + @ParameterizedTest(name = "{0}") + @MethodSource("additionalTestCases") + void given_AsyncPresignedUrlExtension_when_ExecutingDifferentScenarios_then_ShouldBehaveCorrectly( + String testName, String testType) throws Exception { + + switch (testType) { + case "CUSTOM_TRANSFORMER": + mockHttpClient.stubNextResponse(createSuccessResponse()); + CompletableFuture> future = + presignedUrlExtension.getObject(testRequest, AsyncResponseTransformer.toBytes()); + ResponseBytes result = future.get(); + assertThat(result.asUtf8String()).isEqualTo(TEST_CONTENT); + break; + + case "METRICS_COLLECTION": + MetricPublisher mockPublisher = mock(MetricPublisher.class); + SdkClientConfiguration clientConfigWithMetrics = getDefaultSdkConfigs().toBuilder() + .option(SdkClientOption.METRIC_PUBLISHERS, Collections.singletonList(mockPublisher)) + .build(); + DefaultAsyncPresignedUrlExtension extensionWithMetrics = new DefaultAsyncPresignedUrlExtension( + clientHandler, protocolFactory, clientConfigWithMetrics, protocolMetadata); + mockHttpClient.stubNextResponse(createSuccessResponse()); + CompletableFuture> metricsFuture = + extensionWithMetrics.getObject(testRequest, AsyncResponseTransformer.toBytes()); + metricsFuture.get(); + + verify(mockPublisher, atLeastOnce()).publish(any(MetricCollection.class)); + ArgumentCaptor metricsCaptor = ArgumentCaptor.forClass(MetricCollection.class); + verify(mockPublisher).publish(metricsCaptor.capture()); + MetricCollection capturedMetrics = metricsCaptor.getValue(); + assertThat(capturedMetrics.metricValues(CoreMetric.SERVICE_ID)).contains("S3"); + assertThat(capturedMetrics.metricValues(CoreMetric.OPERATION_NAME)).contains("PresignedUrlDownload"); + break; + } + } + + @ParameterizedTest(name = "{0}") + @MethodSource("invalidUrlTestCases") + void given_AsyncPresignedUrlExtension_when_GetObjectWithInvalidUrl_then_ShouldThrowException( + String testName, String invalidUrlString) { + assertThatThrownBy(() -> { + URL invalidUrl = new URL(invalidUrlString); + PresignedUrlDownloadRequest invalidRequest = PresignedUrlDownloadRequest.builder() + .presignedUrl(invalidUrl) + .build(); + presignedUrlExtension.getObject(invalidRequest, AsyncResponseTransformer.toBytes()).join(); + }).satisfiesAnyOf( + ex -> assertThat(ex).isInstanceOf(java.net.MalformedURLException.class), + ex -> assertThat(ex).isInstanceOf(SdkClientException.class), + ex -> assertThat(ex).isInstanceOf(java.util.concurrent.CompletionException.class) + .extracting(Throwable::getCause) + .isInstanceOf(SdkClientException.class) + ); + } + + private SdkClientConfiguration getDefaultSdkConfigs() { + return SdkClientConfiguration.builder() + .option(SdkClientOption.ASYNC_HTTP_CLIENT, mockHttpClient) + .option(SdkClientOption.ADDITIONAL_HTTP_HEADERS, Collections.emptyMap()) + .option(SdkClientOption.EXECUTION_INTERCEPTORS, Collections.emptyList()) + .option(SdkClientOption.RETRY_STRATEGY, AwsRetryStrategy.doNotRetry()) + .option(SdkAdvancedClientOption.USER_AGENT_PREFIX, "") + .option(SdkAdvancedClientOption.USER_AGENT_SUFFIX, "") + .option(SdkClientOption.CRC32_FROM_COMPRESSED_DATA_ENABLED, false) + .option(AwsClientOption.CREDENTIALS_IDENTITY_PROVIDER, AnonymousCredentialsProvider.create()) + .option(AwsClientOption.AWS_REGION, Region.US_EAST_2) + .option(AwsClientOption.SIGNING_REGION, Region.US_EAST_2) + .option(AwsClientOption.SERVICE_SIGNING_NAME, Region.AP_EAST_2.toString()) + .option(AwsAdvancedClientOption.ENABLE_DEFAULT_REGION_DETECTION, false) + .option(SdkClientOption.SCHEDULED_EXECUTOR_SERVICE, Executors.newScheduledThreadPool(1)) + .option(SdkAdvancedAsyncClientOption.FUTURE_COMPLETION_EXECUTOR, Runnable::run) + .option(SdkClientOption.CLIENT_ENDPOINT_PROVIDER, + software.amazon.awssdk.core.ClientEndpointProvider.forEndpointOverride(DEFAULT_ENDPOINT)) + .build(); + } + + private AwsS3ProtocolFactory initProtocolFactory(SdkClientConfiguration configuration) { + return AwsS3ProtocolFactory.builder() + .registerModeledException( + ExceptionMetadata.builder().errorCode("NoSuchKey") + .exceptionBuilderSupplier(NoSuchKeyException::builder) + .httpStatusCode(404).build()) + .registerModeledException( + ExceptionMetadata.builder().errorCode("InvalidObjectState") + .exceptionBuilderSupplier(InvalidObjectStateException::builder) + .httpStatusCode(403).build()) + .clientConfiguration(configuration) + .defaultServiceExceptionSupplier(S3Exception::builder) + .build(); + } + + private static URL createTestUrl() { + try { + return new URL(TEST_URL); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + private void assertSuccessfulGetObject(PresignedUrlDownloadRequest request) { + try { + CompletableFuture> future = + presignedUrlExtension.getObject(request, AsyncResponseTransformer.toBytes()); + ResponseBytes result = future.get(); + + assertThat(result).isNotNull(); + assertThat(result.asUtf8String()).isEqualTo(TEST_CONTENT); + + SdkHttpRequest lastRequest = mockHttpClient.getLastRequest(); + assertThat(lastRequest.method()).isEqualTo(SdkHttpMethod.GET); + assertThat(lastRequest.getUri().toString()).contains("test-bucket.s3.us-east-1.amazonaws.com/test-key"); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + @ParameterizedTest(name = "{0}") + @MethodSource("checksumValidationTestCases") + void getObject_withChecksumHeaderInResponse_shouldDownloadSuccessfully( + String testName, + HttpExecuteResponse response, + String testUrl, + boolean expectSuccess) throws Exception { + + mockHttpClient.stubNextResponse(response); + + URL presignedUrl = new URL(testUrl); + PresignedUrlDownloadRequest request = PresignedUrlDownloadRequest.builder() + .presignedUrl(presignedUrl) + .build(); + + CompletableFuture> future = + presignedUrlExtension.getObject(request, AsyncResponseTransformer.toBytes()); + + ResponseBytes result = future.get(); + assertThat(result).isNotNull(); + assertThat(result.asUtf8String()).isEqualTo(TEST_CONTENT); + } + + private static Stream checksumValidationTestCases() { + // CRC32 of "test-content" = 0x6B59FCDE → base64 = a1n83g== + String correctCrc32 = "a1n83g=="; + String urlWithChecksumSigned = "https://test-bucket.s3.us-east-1.amazonaws.com/test-key?" + + "X-Amz-Algorithm=AWS4-HMAC-SHA256&" + + "X-Amz-Date=20250707T000000Z&" + + "X-Amz-SignedHeaders=host%3Bx-amz-checksum-mode&" + + "X-Amz-Signature=test-signature&" + + "X-Amz-Credential=AKIAIOSFODNN7EXAMPLE%2F20250707%2Fus-east-1%2Fs3%2Faws4_request&" + + "X-Amz-Expires=600"; + + return Stream.of( + Arguments.of( + "With checksum in response - should succeed", + createResponseWithChecksum("x-amz-checksum-crc32", correctCrc32), + urlWithChecksumSigned, + true + ), + Arguments.of( + "No checksum header in response - should succeed without validation", + createSuccessResponse(), + TEST_URL, + true + ) + ); + } + + private static HttpExecuteResponse createResponseWithChecksum(String checksumHeader, String checksumValue) { + SdkHttpFullResponse httpResponse = SdkHttpFullResponse.builder() + .statusCode(200) + .putHeader("Content-Length", "12") + .putHeader("ETag", "\"test-etag\"") + .putHeader("Content-Type", "text/plain") + .putHeader(checksumHeader, checksumValue) + .build(); + return HttpExecuteResponse.builder() + .response(httpResponse) + .responseBody(AbortableInputStream.create( + new ByteArrayInputStream(TEST_CONTENT.getBytes(StandardCharsets.UTF_8)))) + .build(); + } + + private static HttpExecuteResponse createSuccessResponse() { + SdkHttpFullResponse httpResponse = SdkHttpFullResponse.builder() + .statusCode(200) + .putHeader("Content-Length", "12") + .putHeader("ETag", "\"test-etag\"") + .putHeader("Content-Type", "text/plain") + .build(); + return HttpExecuteResponse.builder() + .response(httpResponse) + .responseBody(AbortableInputStream.create( + new ByteArrayInputStream(TEST_CONTENT.getBytes(StandardCharsets.UTF_8)))) + .build(); + } + + private static HttpExecuteResponse createErrorResponse(int statusCode, String errorCode, String errorMessage) { + String errorContent = String.format( + "%s%s", + errorCode, errorMessage); + SdkHttpFullResponse httpResponse = SdkHttpFullResponse.builder() + .statusCode(statusCode) + .putHeader("x-amz-request-id", "test-request-id") + .putHeader("x-amz-id-2", "test-extended-request-id") + .build(); + return HttpExecuteResponse.builder() + .response(httpResponse) + .responseBody(AbortableInputStream.create( + new ByteArrayInputStream(errorContent.getBytes(StandardCharsets.UTF_8)))) + .build(); + } +} \ No newline at end of file diff --git a/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/presignedurl/PresignedUrlChecksumValidationWiremockTest.java b/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/presignedurl/PresignedUrlChecksumValidationWiremockTest.java new file mode 100644 index 000000000000..fc2a7a26cf97 --- /dev/null +++ b/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/presignedurl/PresignedUrlChecksumValidationWiremockTest.java @@ -0,0 +1,216 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.services.s3.internal.presignedurl; + +import static com.github.tomakehurst.wiremock.client.WireMock.aResponse; +import static com.github.tomakehurst.wiremock.client.WireMock.get; +import static com.github.tomakehurst.wiremock.client.WireMock.stubFor; +import static com.github.tomakehurst.wiremock.client.WireMock.urlPathEqualTo; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import com.github.tomakehurst.wiremock.junit5.WireMockRuntimeInfo; +import com.github.tomakehurst.wiremock.junit5.WireMockTest; +import java.net.URL; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.atomic.AtomicReference; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import software.amazon.awssdk.auth.credentials.AnonymousCredentialsProvider; +import software.amazon.awssdk.core.ResponseBytes; +import software.amazon.awssdk.core.async.AsyncResponseTransformer; +import software.amazon.awssdk.core.checksums.ChecksumValidation; +import software.amazon.awssdk.core.exception.SdkClientException; +import software.amazon.awssdk.core.interceptor.Context; +import software.amazon.awssdk.core.interceptor.ExecutionAttributes; +import software.amazon.awssdk.core.interceptor.ExecutionInterceptor; +import software.amazon.awssdk.core.interceptor.SdkExecutionAttribute; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.s3.S3AsyncClient; +import software.amazon.awssdk.services.s3.model.GetObjectResponse; +import software.amazon.awssdk.services.s3.presignedurl.model.PresignedUrlDownloadRequest; + +/** + * WireMock test verifying checksum validation behavior for presigned URL downloads. + */ +@WireMockTest +class PresignedUrlChecksumValidationWiremockTest { + + private static final String BODY = "test-content-for-checksum"; + private static final String CORRECT_CRC32 = "HUUjuQ=="; // CRC32 of "test-content-for-checksum" + private static final String INCORRECT_CRC32 = "AAAAAAAA=="; + + private S3AsyncClient s3Client; + private final AtomicReference checksumValidationStatus = new AtomicReference<>(); + + @BeforeEach + void setUp(WireMockRuntimeInfo wmInfo) { + checksumValidationStatus.set(null); + s3Client = S3AsyncClient.builder() + .endpointOverride(java.net.URI.create(wmInfo.getHttpBaseUrl())) + .region(Region.US_EAST_1) + .credentialsProvider(AnonymousCredentialsProvider.create()) + .forcePathStyle(true) + .overrideConfiguration(c -> c.addExecutionInterceptor(new ExecutionInterceptor() { + @Override + public void afterExecution(Context.AfterExecution context, ExecutionAttributes attrs) { + checksumValidationStatus.set( + attrs.getAttribute(SdkExecutionAttribute.HTTP_RESPONSE_CHECKSUM_VALIDATION)); + } + })) + .build(); + } + + @AfterEach + void tearDown() { + if (s3Client != null) { + s3Client.close(); + } + } + + @Test + void getObject_withMatchingChecksum_shouldSucceed(WireMockRuntimeInfo wmInfo) throws Exception { + stubFor(get(urlPathEqualTo("/test-key")) + .willReturn(aResponse() + .withStatus(200) + .withHeader("Content-Length", String.valueOf(BODY.length())) + .withHeader("x-amz-checksum-crc32", CORRECT_CRC32) + .withHeader("x-amz-checksum-type", "FULL_OBJECT") + .withBody(BODY))); + + URL presignedUrl = new URL(wmInfo.getHttpBaseUrl() + "/test-key?" + + "X-Amz-Algorithm=AWS4-HMAC-SHA256&" + + "X-Amz-SignedHeaders=host%3Bx-amz-checksum-mode&" + + "X-Amz-Signature=fake&" + + "X-Amz-Expires=600"); + + ResponseBytes result = s3Client.presignedUrlExtension() + .getObject( + PresignedUrlDownloadRequest.builder().presignedUrl(presignedUrl).build(), + AsyncResponseTransformer.toBytes()) + .join(); + + assertThat(result.asUtf8String()).isEqualTo(BODY); + assertThat(checksumValidationStatus.get()) + .isEqualTo(ChecksumValidation.VALIDATED); + } + + @Test + void getObject_withMismatchingChecksum_shouldThrow(WireMockRuntimeInfo wmInfo) throws Exception { + stubFor(get(urlPathEqualTo("/test-key")) + .willReturn(aResponse() + .withStatus(200) + .withHeader("Content-Length", String.valueOf(BODY.length())) + .withHeader("x-amz-checksum-crc32", INCORRECT_CRC32) + .withHeader("x-amz-checksum-type", "FULL_OBJECT") + .withBody(BODY))); + + URL presignedUrl = new URL(wmInfo.getHttpBaseUrl() + "/test-key?" + + "X-Amz-Algorithm=AWS4-HMAC-SHA256&" + + "X-Amz-SignedHeaders=host%3Bx-amz-checksum-mode&" + + "X-Amz-Signature=fake&" + + "X-Amz-Expires=600"); + + CompletableFuture> future = s3Client.presignedUrlExtension() + .getObject( + PresignedUrlDownloadRequest.builder().presignedUrl(presignedUrl).build(), + AsyncResponseTransformer.toBytes()); + + assertThatThrownBy(future::join) + .hasCauseInstanceOf(SdkClientException.class) + .hasMessageContaining("checksum"); + } + + @Test + void getObject_withNoChecksumHeader_shouldSucceedWithoutValidation(WireMockRuntimeInfo wmInfo) throws Exception { + stubFor(get(urlPathEqualTo("/test-key")) + .willReturn(aResponse() + .withStatus(200) + .withHeader("Content-Length", String.valueOf(BODY.length())) + .withBody(BODY))); + + URL presignedUrl = new URL(wmInfo.getHttpBaseUrl() + "/test-key?" + + "X-Amz-Algorithm=AWS4-HMAC-SHA256&" + + "X-Amz-SignedHeaders=host&" + + "X-Amz-Signature=fake&" + + "X-Amz-Expires=600"); + + ResponseBytes result = s3Client.presignedUrlExtension() + .getObject( + PresignedUrlDownloadRequest.builder().presignedUrl(presignedUrl).build(), + AsyncResponseTransformer.toBytes()) + .join(); + + assertThat(result.asUtf8String()).isEqualTo(BODY); + assertThat(checksumValidationStatus.get()).isNull(); + } + + @Test + void getObject_withChecksumModeButNoChecksumInResponse_shouldSetAlgorithmNotFound(WireMockRuntimeInfo wmInfo) + throws Exception { + // URL has checksum mode signed, but S3 response has no checksum header (e.g., ranged GET on single-PUT) + stubFor(get(urlPathEqualTo("/test-key")) + .willReturn(aResponse() + .withStatus(200) + .withHeader("Content-Length", String.valueOf(BODY.length())) + .withBody(BODY))); + + URL presignedUrl = new URL(wmInfo.getHttpBaseUrl() + "/test-key?" + + "X-Amz-Algorithm=AWS4-HMAC-SHA256&" + + "X-Amz-SignedHeaders=host%3Bx-amz-checksum-mode&" + + "X-Amz-Signature=fake&" + + "X-Amz-Expires=600"); + + ResponseBytes result = s3Client.presignedUrlExtension() + .getObject( + PresignedUrlDownloadRequest.builder().presignedUrl(presignedUrl).build(), + AsyncResponseTransformer.toBytes()) + .join(); + + assertThat(result.asUtf8String()).isEqualTo(BODY); + assertThat(checksumValidationStatus.get()) + .isEqualTo(ChecksumValidation.CHECKSUM_ALGORITHM_NOT_FOUND); + } + + @Test + void getObject_withCompositeChecksum_shouldSkipValidationAndSucceed(WireMockRuntimeInfo wmInfo) throws Exception { + stubFor(get(urlPathEqualTo("/test-key")) + .willReturn(aResponse() + .withStatus(200) + .withHeader("Content-Length", String.valueOf(BODY.length())) + .withHeader("x-amz-checksum-crc32", "i0T6cA==-3") + .withHeader("x-amz-checksum-type", "COMPOSITE") + .withBody(BODY))); + + URL presignedUrl = new URL(wmInfo.getHttpBaseUrl() + "/test-key?" + + "X-Amz-Algorithm=AWS4-HMAC-SHA256&" + + "X-Amz-SignedHeaders=host%3Bx-amz-checksum-mode&" + + "X-Amz-Signature=fake&" + + "X-Amz-Expires=600"); + + ResponseBytes result = s3Client.presignedUrlExtension() + .getObject( + PresignedUrlDownloadRequest.builder().presignedUrl(presignedUrl).build(), + AsyncResponseTransformer.toBytes()) + .join(); + + assertThat(result.asUtf8String()).isEqualTo(BODY); + assertThat(checksumValidationStatus.get()) + .isEqualTo(ChecksumValidation.FORCE_SKIP); + + } +} diff --git a/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/presignedurl/PresignedUrlDownloadRequestMarshallerTest.java b/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/presignedurl/PresignedUrlDownloadRequestMarshallerTest.java new file mode 100644 index 000000000000..04c047cb4cfa --- /dev/null +++ b/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/presignedurl/PresignedUrlDownloadRequestMarshallerTest.java @@ -0,0 +1,244 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.services.s3.internal.presignedurl; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; +import java.net.URL; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.NullAndEmptySource; +import org.junit.jupiter.params.provider.ValueSource; +import software.amazon.awssdk.core.exception.SdkClientException; +import software.amazon.awssdk.http.SdkHttpFullRequest; +import software.amazon.awssdk.http.SdkHttpMethod; +import software.amazon.awssdk.protocols.core.OperationInfo; +import software.amazon.awssdk.protocols.core.ProtocolMarshaller; +import software.amazon.awssdk.protocols.xml.AwsXmlProtocolFactory; +import software.amazon.awssdk.services.s3.internal.presignedurl.model.PresignedUrlDownloadRequestWrapper; + +class PresignedUrlDownloadRequestMarshallerTest { + + private PresignedUrlDownloadRequestMarshaller marshaller; + private AwsXmlProtocolFactory mockProtocolFactory; + private ProtocolMarshaller mockProtocolMarshaller; + private URL testUrl; + + @BeforeEach + void setUp() throws Exception { + mockProtocolFactory = mock(AwsXmlProtocolFactory.class); + mockProtocolMarshaller = mock(ProtocolMarshaller.class); + when(mockProtocolFactory.createProtocolMarshaller(any(OperationInfo.class))) + .thenReturn(mockProtocolMarshaller); + marshaller = new PresignedUrlDownloadRequestMarshaller(mockProtocolFactory); + + testUrl = new URL("https://test-bucket.s3.us-east-1.amazonaws.com/test-key?" + + "X-Amz-Date=20231215T000000Z&" + + "X-Amz-Signature=example-signature&" + + "X-Amz-Algorithm=AWS4-HMAC-SHA256&" + + "X-Amz-SignedHeaders=host&" + + "X-Amz-Security-Token=xxx&" + + "X-Amz-Credential=EXAMPLE12345678901234%2F20231215%2Fus-east-1%2Fs3%2Faws4_request&" + + "X-Amz-Expires=3600"); + } + + @Test + void marshall_withBasicRequest_shouldCreateCorrectHttpRequest() throws Exception { + // Setup the mock marshaller to return a properly configured request + SdkHttpFullRequest baseRequest = SdkHttpFullRequest.builder() + .method(SdkHttpMethod.GET) + .protocol("https") + .host("example.com") + .build(); + when(mockProtocolMarshaller.marshall(any(PresignedUrlDownloadRequestWrapper.class))) + .thenReturn(baseRequest); + + PresignedUrlDownloadRequestWrapper request = PresignedUrlDownloadRequestWrapper.builder() + .url(testUrl) + .build(); + SdkHttpFullRequest result = marshaller.marshall(request); + + // Verify HTTP method and URI components + assertThat(result.method()).isEqualTo(SdkHttpMethod.GET); + assertThat(result.getUri()) + .satisfies(uri -> { + assertThat(uri.getScheme()).isEqualTo("https"); + assertThat(uri.getHost()).isEqualTo("test-bucket.s3.us-east-1.amazonaws.com"); + assertThat(uri.getPath()).isEqualTo("/test-key"); + }); + + // Verify query parameters are preserved + assertThat(result.getUri().getQuery()) + .contains("X-Amz-Date=20231215T000000Z") + .contains("X-Amz-Signature=example-signature") + .contains("X-Amz-Algorithm=AWS4-HMAC-SHA256") + .contains("X-Amz-SignedHeaders=host") + .contains("X-Amz-Security-Token=xxx") + .contains("X-Amz-Credential=EXAMPLE12345678901234") + .contains("X-Amz-Expires=3600"); + + assertThat(result.headers()).doesNotContainKey("Range"); + } + + @ParameterizedTest + @ValueSource(strings = { + "bytes=0-100", // First 101 bytes + "bytes=100-", // From byte 100 to end + "bytes=-100", // Last 100 bytes + "bytes=0-0", // Single byte + "bytes=100-200" // Specific range + }) + void marshall_withValidRangeFormats_shouldAddRangeHeader(String rangeValue) throws Exception { + // Setup the mock marshaller to return a request with the Range header already set + SdkHttpFullRequest baseRequest = SdkHttpFullRequest.builder() + .method(SdkHttpMethod.GET) + .protocol("https") + .host("example.com") + .putHeader("Range", rangeValue) // Add the Range header to the mock response + .build(); + + when(mockProtocolMarshaller.marshall(any(PresignedUrlDownloadRequestWrapper.class))) + .thenReturn(baseRequest); + + PresignedUrlDownloadRequestWrapper request = PresignedUrlDownloadRequestWrapper.builder() + .url(testUrl) + .range(rangeValue) + .build(); + + SdkHttpFullRequest result = marshaller.marshall(request); + + // Verify the Range header is preserved + assertThat(result.headers()) + .containsKey("Range") + .satisfies(headers -> assertThat(headers.get("Range")).contains(rangeValue)); + } + + @ParameterizedTest + @NullAndEmptySource + void marshall_withNullOrEmptyRange_shouldNotAddRangeHeader(String rangeValue) throws Exception { + // Setup the mock marshaller to return a properly configured request + SdkHttpFullRequest baseRequest = SdkHttpFullRequest.builder() + .method(SdkHttpMethod.GET) + .protocol("https") + .host("example.com") + .build(); + when(mockProtocolMarshaller.marshall(any(PresignedUrlDownloadRequestWrapper.class))) + .thenReturn(baseRequest); + + PresignedUrlDownloadRequestWrapper request = PresignedUrlDownloadRequestWrapper.builder() + .url(testUrl) + .range(rangeValue) + .build(); + SdkHttpFullRequest result = marshaller.marshall(request); + + assertThat(result.headers()).doesNotContainKey("Range"); + } + + @Test + void marshall_withNullRequest_shouldThrowException() { + assertThatThrownBy(() -> marshaller.marshall(null)) + .isInstanceOf(NullPointerException.class) + .hasMessageContaining("presignedUrlDownloadRequestWrapper must not be null"); + } + + @Test + void marshall_withChecksumModeInSignedHeaders_shouldAddChecksumHeader() throws Exception { + URL urlWithChecksum = new URL("https://test-bucket.s3.us-east-1.amazonaws.com/test-key?" + + "X-Amz-Algorithm=AWS4-HMAC-SHA256&" + + "X-Amz-SignedHeaders=host%3Bx-amz-checksum-mode&" + + "X-Amz-Signature=abc123&" + + "X-Amz-Expires=600"); + + SdkHttpFullRequest baseRequest = SdkHttpFullRequest.builder() + .method(SdkHttpMethod.GET) + .protocol("https") + .host("example.com") + .build(); + when(mockProtocolMarshaller.marshall(any(PresignedUrlDownloadRequestWrapper.class))) + .thenReturn(baseRequest); + + PresignedUrlDownloadRequestWrapper request = PresignedUrlDownloadRequestWrapper.builder() + .url(urlWithChecksum) + .build(); + SdkHttpFullRequest result = marshaller.marshall(request); + + assertThat(result.headers()).containsKey("x-amz-checksum-mode"); + assertThat(result.firstMatchingHeader("x-amz-checksum-mode")).hasValue("ENABLED"); + } + + @Test + void marshall_withoutChecksumModeInSignedHeaders_shouldNotAddChecksumHeader() throws Exception { + SdkHttpFullRequest baseRequest = SdkHttpFullRequest.builder() + .method(SdkHttpMethod.GET) + .protocol("https") + .host("example.com") + .build(); + when(mockProtocolMarshaller.marshall(any(PresignedUrlDownloadRequestWrapper.class))) + .thenReturn(baseRequest); + + PresignedUrlDownloadRequestWrapper request = PresignedUrlDownloadRequestWrapper.builder() + .url(testUrl) + .build(); + SdkHttpFullRequest result = marshaller.marshall(request); + + assertThat(result.headers()).doesNotContainKey("x-amz-checksum-mode"); + } + + @Test + void marshall_withNoQueryString_shouldNotAddChecksumHeader() throws Exception { + URL urlNoQuery = new URL("https://test-bucket.s3.us-east-1.amazonaws.com/test-key"); + + SdkHttpFullRequest baseRequest = SdkHttpFullRequest.builder() + .method(SdkHttpMethod.GET) + .protocol("https") + .host("example.com") + .build(); + when(mockProtocolMarshaller.marshall(any(PresignedUrlDownloadRequestWrapper.class))) + .thenReturn(baseRequest); + + PresignedUrlDownloadRequestWrapper request = PresignedUrlDownloadRequestWrapper.builder() + .url(urlNoQuery) + .build(); + SdkHttpFullRequest result = marshaller.marshall(request); + + assertThat(result.headers()).doesNotContainKey("x-amz-checksum-mode"); + } + + @Test + void marshall_withMalformedUrl_shouldThrowSdkClientException() throws Exception { + // Setup the mock marshaller to return a properly configured request + SdkHttpFullRequest baseRequest = SdkHttpFullRequest.builder() + .method(SdkHttpMethod.GET) + .protocol("https") + .host("example.com") + .build(); + when(mockProtocolMarshaller.marshall(any(PresignedUrlDownloadRequestWrapper.class))) + .thenReturn(baseRequest); + + URL malformedUrl = new URL("https", "test-bucket.s3.us-east-1.amazonaws.com", -1, "/test key with spaces"); + PresignedUrlDownloadRequestWrapper request = PresignedUrlDownloadRequestWrapper.builder() + .url(malformedUrl) + .build(); + + assertThatThrownBy(() -> marshaller.marshall(request)) + .isInstanceOf(SdkClientException.class) + .hasMessageContaining("Unable to marshall pre-signed URL Request"); + } +} \ No newline at end of file diff --git a/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/presignedurl/model/PresignedUrlDownloadRequestWrapperTest.java b/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/presignedurl/model/PresignedUrlDownloadRequestWrapperTest.java new file mode 100644 index 000000000000..8b301a332199 --- /dev/null +++ b/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/presignedurl/model/PresignedUrlDownloadRequestWrapperTest.java @@ -0,0 +1,118 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.services.s3.internal.presignedurl.model; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.net.URL; +import java.util.List; +import java.util.Map; +import nl.jqno.equalsverifier.EqualsVerifier; +import org.junit.jupiter.api.Test; +import software.amazon.awssdk.core.SdkField; +import software.amazon.awssdk.core.protocol.MarshallLocation; + + +class PresignedUrlDownloadRequestWrapperTest { + @Test + void equalsAndHashCode_shouldFollowContract() { + EqualsVerifier.forClass(PresignedUrlDownloadRequestWrapper.class) + .withRedefinedSuperclass() + .verify(); + } + + @Test + void basicProperties_shouldWork() throws Exception { + PresignedUrlDownloadRequestWrapper request = PresignedUrlDownloadRequestWrapper.builder() + .url(new URL("https://example.com")) + .range("bytes=0-100") + .ifMatch("\"etag-123\"") + .build(); + + assertThat(request.url()).isEqualTo(new URL("https://example.com")); + assertThat(request.range()).isEqualTo("bytes=0-100"); + assertThat(request.ifMatch()).isEqualTo("\"etag-123\""); + } + + @Test + void sdkFields_shouldReturnExpectedFields() throws Exception { + PresignedUrlDownloadRequestWrapper request = PresignedUrlDownloadRequestWrapper.builder() + .url(new URL("https://example.com")) + .range("bytes=0-100") + .build(); + + List> fields = request.sdkFields(); + + assertThat(fields).hasSize(2); + assertThat(fields).extracting(SdkField::memberName) + .containsExactlyInAnyOrder("Range", "IfMatch"); + assertThat(fields).allMatch(field -> field.location() == MarshallLocation.HEADER); + SdkField rangeField = fields.stream() + .filter(f -> "Range".equals(f.memberName())) + .findFirst() + .orElseThrow(() -> new AssertionError("Range field not found")); + Object rangeValue = rangeField.getValueOrDefault(request); + assertThat(rangeValue).isNotNull(); + assertThat(rangeValue).isEqualTo("bytes=0-100"); + } + + @Test + void sdkFieldNameToField_shouldReturnExpectedMapping() throws Exception { + PresignedUrlDownloadRequestWrapper request = PresignedUrlDownloadRequestWrapper.builder() + .url(new URL("https://example.com")) + .build(); + + Map> fieldMap = request.sdkFieldNameToField(); + + assertThat(fieldMap) + .hasSize(2) + .containsKeys("Range", "IfMatch"); + assertThat(fieldMap.get("Range").memberName()).isEqualTo("Range"); + assertThat(fieldMap.get("IfMatch").memberName()).isEqualTo("IfMatch"); + } + + @Test + void rangeField_shouldMarshalCorrectly() throws Exception { + PresignedUrlDownloadRequestWrapper request = PresignedUrlDownloadRequestWrapper.builder() + .url(new URL("https://example.com")) + .range("bytes=0-1023") + .build(); + + SdkField rangeField = request.sdkFields().stream() + .filter(f -> "Range".equals(f.memberName())) + .findFirst() + .orElseThrow(() -> new AssertionError("Range field not found")); + Object extractedValue = rangeField.getValueOrDefault(request); + + assertThat(extractedValue).isEqualTo("bytes=0-1023"); + } + + @Test + void ifMatchField_shouldMarshalCorrectly() throws Exception { + PresignedUrlDownloadRequestWrapper request = PresignedUrlDownloadRequestWrapper.builder() + .url(new URL("https://example.com")) + .ifMatch("\"etag-value\"") + .build(); + + SdkField ifMatchField = request.sdkFields().stream() + .filter(f -> "IfMatch".equals(f.memberName())) + .findFirst() + .orElseThrow(() -> new AssertionError("IfMatch field not found")); + Object extractedValue = ifMatchField.getValueOrDefault(request); + + assertThat(extractedValue).isEqualTo("\"etag-value\""); + } +} diff --git a/services/s3/src/test/java/software/amazon/awssdk/services/s3/presignedurl/AsyncPresignedUrlExtensionTest.java b/services/s3/src/test/java/software/amazon/awssdk/services/s3/presignedurl/AsyncPresignedUrlExtensionTest.java new file mode 100644 index 000000000000..1b99c9bde491 --- /dev/null +++ b/services/s3/src/test/java/software/amazon/awssdk/services/s3/presignedurl/AsyncPresignedUrlExtensionTest.java @@ -0,0 +1,522 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.services.s3.presignedurl; + +import static com.github.tomakehurst.wiremock.client.WireMock.aResponse; +import static com.github.tomakehurst.wiremock.client.WireMock.exactly; +import static com.github.tomakehurst.wiremock.client.WireMock.get; +import static com.github.tomakehurst.wiremock.client.WireMock.getRequestedFor; +import static com.github.tomakehurst.wiremock.client.WireMock.stubFor; +import static com.github.tomakehurst.wiremock.client.WireMock.urlEqualTo; +import static com.github.tomakehurst.wiremock.client.WireMock.verify; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import java.net.URI; +import java.net.URL; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; +import java.util.Random; +import java.util.UUID; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; + +import com.github.tomakehurst.wiremock.http.Fault; +import com.github.tomakehurst.wiremock.junit5.WireMockRuntimeInfo; +import com.github.tomakehurst.wiremock.junit5.WireMockTest; +import com.github.tomakehurst.wiremock.stubbing.Scenario; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import software.amazon.awssdk.core.ResponseBytes; +import software.amazon.awssdk.core.async.AsyncResponseTransformer; +import software.amazon.awssdk.core.exception.SdkClientException; +import software.amazon.awssdk.services.s3.S3AsyncClient; +import software.amazon.awssdk.services.s3.model.GetObjectResponse; +import software.amazon.awssdk.services.s3.presignedurl.model.PresignedUrlDownloadRequest; + +@WireMockTest +public class AsyncPresignedUrlExtensionTest { + + private S3AsyncClient s3AsyncClient; + private AsyncPresignedUrlExtension presignedUrlExtension; + + @TempDir + Path tempDir; + + @BeforeEach + void setup(WireMockRuntimeInfo wireMockRuntimeInfo) { + s3AsyncClient = S3AsyncClient.builder() + .endpointOverride(URI.create(wireMockRuntimeInfo.getHttpBaseUrl())) + .build(); + presignedUrlExtension = s3AsyncClient.presignedUrlExtension(); + } + + @AfterEach + void cleanup() { + if (s3AsyncClient != null) { + s3AsyncClient.close(); + } + } + + @Nested + class BasicFunctionality { + @Test + void givenS3AsyncClient_whenPresignedUrlExtensionRequested_thenReturnsNonNullInstance() { + assertThat(presignedUrlExtension).isNotNull(); + } + + @Test + void givenValidPresignedUrl_whenGetObjectCalled_thenReturnsExpectedContent(WireMockRuntimeInfo wireMockRuntimeInfo) throws Exception { + String testContent = "Hello world"; + String testETag = "\"d6eb32081c822ed572b70567826d9d9d\""; + String presignedUrlPath = "/presigned-test-object"; + + stubSuccessResponse(presignedUrlPath, testContent, testETag); + + URL presignedUrl = createPresignedUrl(wireMockRuntimeInfo, presignedUrlPath); + PresignedUrlDownloadRequest request = createRequest(presignedUrl); + + CompletableFuture> result = + presignedUrlExtension.getObject(request, AsyncResponseTransformer.toBytes()); + + ResponseBytes response = result.get(); + assertSuccessfulResponse(response, testContent, testETag); + } + + @Test + void givenValidPresignedUrl_whenGetObjectWithConsumerBuilderCalled_thenReturnsExpectedContent(WireMockRuntimeInfo wireMockRuntimeInfo) throws Exception { + String testContent = "Hello consumer"; + String testETag = "\"c1234567890abcdef1234567890abcdef\""; + String presignedUrlPath = "/presigned-test-consumer"; + + stubSuccessResponse(presignedUrlPath, testContent, testETag); + + URL presignedUrl = createPresignedUrl(wireMockRuntimeInfo, presignedUrlPath); + + CompletableFuture> result = + presignedUrlExtension.getObject( + request -> request.presignedUrl(presignedUrl), + AsyncResponseTransformer.toBytes() + ); + + ResponseBytes response = result.get(); + assertSuccessfulResponse(response, testContent, testETag); + } + + @Test + void givenEmptyFile_whenGetObjectCalled_thenReturnsEmptyContent(WireMockRuntimeInfo wireMockRuntimeInfo) throws Exception { + String testContent = ""; + String testETag = "\"empty-file-etag\""; + String presignedUrlPath = "/empty-file-memory"; + + // Using custom stubbing for empty file to include Content-Length header + stubFor(get(urlEqualTo(presignedUrlPath)) + .willReturn(aResponse() + .withStatus(200) + .withHeader("ETag", testETag) + .withHeader("Content-Length", "0") + .withBody(testContent))); + + URL presignedUrl = createPresignedUrl(wireMockRuntimeInfo, presignedUrlPath); + PresignedUrlDownloadRequest request = createRequest(presignedUrl); + + CompletableFuture> result = + presignedUrlExtension.getObject(request, AsyncResponseTransformer.toBytes()); + + ResponseBytes response = result.get(); + assertThat(response).isNotNull(); + assertThat(response.asByteArray()).isEmpty(); + assertThat(response.response().eTag()).isEqualTo(testETag); + } + + @Test + void givenRangeRequest_whenGetObjectCalled_thenReturnsPartialContent(WireMockRuntimeInfo wireMockRuntimeInfo) throws Exception { + String expectedContent = "Hello"; + String contentRange = "bytes 0-4/11"; + String range = "bytes=0-4"; + String testETag = "\"d6eb32081c822ed572b70567826d9d9d\""; + String presignedUrlPath = "/presigned-test-range"; + + // Custom stubbing for range request with 206 status + stubFor(get(urlEqualTo(presignedUrlPath)) + .willReturn(aResponse() + .withStatus(206) + .withHeader("ETag", testETag) + .withHeader("Content-Range", contentRange) + .withBody(expectedContent))); + + URL presignedUrl = createPresignedUrl(wireMockRuntimeInfo, presignedUrlPath); + PresignedUrlDownloadRequest request = PresignedUrlDownloadRequest.builder() + .presignedUrl(presignedUrl) + .range(range) + .build(); + + CompletableFuture> result = + presignedUrlExtension.getObject(request, AsyncResponseTransformer.toBytes()); + + ResponseBytes response = result.get(); + assertThat(response).isNotNull(); + assertThat(response.asUtf8String()).isEqualTo(expectedContent); + } + } + + @Nested + class FileDownloads { + @Test + void givenValidPresignedUrl_whenGetObjectToFileCalled_thenDownloadsFileWithExpectedContent(WireMockRuntimeInfo wireMockRuntimeInfo) throws Exception { + String testContent = "File content for download"; + String testETag = generateRandomETag(); + String presignedUrlPath = "/presigned-test-file"; + + stubSuccessResponse(presignedUrlPath, testContent, testETag); + + URL presignedUrl = createPresignedUrl(wireMockRuntimeInfo, presignedUrlPath); + PresignedUrlDownloadRequest request = createRequest(presignedUrl); + + Path downloadFile = tempDir.resolve("download-" + UUID.randomUUID() + ".txt"); + + CompletableFuture result = + presignedUrlExtension.getObject(request, downloadFile); + + GetObjectResponse response = result.get(); + assertThat(response).isNotNull(); + assertThat(response.eTag()).isEqualTo(testETag); + + String fileContent = new String(Files.readAllBytes(downloadFile), StandardCharsets.UTF_8); + assertThat(fileContent).isEqualTo(testContent); + } + + @Test + void givenValidPresignedUrl_whenGetObjectToFileWithConsumerBuilderCalled_thenDownloadsFileWithExpectedContent(WireMockRuntimeInfo wireMockRuntimeInfo) throws Exception { + String testContent = "File content for consumer download"; + String testETag = generateRandomETag(); + String presignedUrlPath = "/presigned-test-consumer-file"; + + stubSuccessResponse(presignedUrlPath, testContent, testETag); + + URL presignedUrl = createPresignedUrl(wireMockRuntimeInfo, presignedUrlPath); + Path downloadFile = tempDir.resolve("consumer-download-" + UUID.randomUUID() + ".txt"); + + CompletableFuture result = + presignedUrlExtension.getObject( + request -> request.presignedUrl(presignedUrl), + downloadFile + ); + + GetObjectResponse response = result.get(); + assertThat(response).isNotNull(); + assertThat(response.eTag()).isEqualTo(testETag); + + String fileContent = new String(Files.readAllBytes(downloadFile), StandardCharsets.UTF_8); + assertThat(fileContent).isEqualTo(testContent); + } + + @Test + void givenEmptyFile_whenGetObjectToFileCalled_thenDownloadsEmptyFile(WireMockRuntimeInfo wireMockRuntimeInfo) throws Exception { + String testContent = ""; + String testETag = "\"empty-file-etag\""; + String presignedUrlPath = "/empty-file"; + + // Custom stubbing for empty file to include Content-Length header + stubFor(get(urlEqualTo(presignedUrlPath)) + .willReturn(aResponse() + .withStatus(200) + .withHeader("ETag", testETag) + .withHeader("Content-Length", "0") + .withBody(testContent))); + + URL presignedUrl = createPresignedUrl(wireMockRuntimeInfo, presignedUrlPath); + PresignedUrlDownloadRequest request = createRequest(presignedUrl); + + Path downloadFile = tempDir.resolve("empty-file-" + UUID.randomUUID() + ".txt"); + CompletableFuture result = presignedUrlExtension.getObject(request, downloadFile); + + GetObjectResponse response = result.get(); + assertThat(response).isNotNull(); + assertThat(response.eTag()).isEqualTo(testETag); + assertThat(Files.size(downloadFile)).isEqualTo(0); + } + + @Test + void givenLargeFile_whenGetObjectCalled_thenDownloadsCompleteContent(WireMockRuntimeInfo wireMockRuntimeInfo) throws Exception { + int contentSize = 1024 * 1024 + 512 * 1024; // 1.5MB + byte[] largeContent = new byte[contentSize]; + new Random().nextBytes(largeContent); + + String testETag = "\"large-file-etag\""; + String presignedUrlPath = "/large-file"; + + // Custom stubbing for large file with Content-Length header + stubFor(get(urlEqualTo(presignedUrlPath)) + .willReturn(aResponse() + .withStatus(200) + .withHeader("ETag", testETag) + .withHeader("Content-Length", String.valueOf(contentSize)) + .withBody(largeContent))); + + URL presignedUrl = createPresignedUrl(wireMockRuntimeInfo, presignedUrlPath); + PresignedUrlDownloadRequest request = createRequest(presignedUrl); + + Path downloadFile = tempDir.resolve("large-file-" + UUID.randomUUID() + ".bin"); + CompletableFuture result = presignedUrlExtension.getObject(request, downloadFile); + + GetObjectResponse response = result.get(); + assertThat(response).isNotNull(); + assertThat(response.eTag()).isEqualTo(testETag); + + assertThat(Files.size(downloadFile)).isEqualTo(contentSize); + + byte[] downloadedContent = Files.readAllBytes(downloadFile); + assertThat(downloadedContent).isEqualTo(largeContent); + } + } + + @Nested + class RetryBehavior { + @Test + void givenNetworkError_whenGetObjectCalled_thenRetriesAndEventuallySucceeds(WireMockRuntimeInfo wireMockRuntimeInfo) throws Exception { + String testContent = "Success after network error"; + String testETag = "\"network-retry-etag\""; + String presignedUrlPath = "/network-error-retry"; + String scenarioName = "network-error-scenario"; + + stubFor(get(urlEqualTo(presignedUrlPath)) + .inScenario(scenarioName) + .whenScenarioStateIs(Scenario.STARTED) + .willReturn(aResponse() + .withFault(Fault.CONNECTION_RESET_BY_PEER)) + .willSetStateTo("after-network-error")); + + stubFor(get(urlEqualTo(presignedUrlPath)) + .inScenario(scenarioName) + .whenScenarioStateIs("after-network-error") + .willReturn(aResponse() + .withStatus(200) + .withHeader("ETag", testETag) + .withBody(testContent))); + + URL presignedUrl = createPresignedUrl(wireMockRuntimeInfo, presignedUrlPath); + PresignedUrlDownloadRequest request = createRequest(presignedUrl); + + CompletableFuture> result = + presignedUrlExtension.getObject(request, AsyncResponseTransformer.toBytes()); + + ResponseBytes response = result.get(); + assertSuccessfulResponse(response, testContent, testETag); + + verify(exactly(2), getRequestedFor(urlEqualTo(presignedUrlPath))); + } + + @Test + void givenTemporaryFailure_whenGetObjectCalled_thenRetriesAndSucceeds(WireMockRuntimeInfo wireMockRuntimeInfo) throws Exception { + String testContent = "Success after retry"; + String testETag = "\"retry-success-etag\""; + String presignedUrlPath = "/retry-test"; + String scenarioName = "retry-scenario"; + + stubFor(get(urlEqualTo(presignedUrlPath)) + .inScenario(scenarioName) + .whenScenarioStateIs(Scenario.STARTED) + .willReturn(aResponse() + .withStatus(503) + .withBody("Service Unavailable")) + .willSetStateTo("retry")); + + stubFor(get(urlEqualTo(presignedUrlPath)) + .inScenario(scenarioName) + .whenScenarioStateIs("retry") + .willReturn(aResponse() + .withStatus(200) + .withHeader("ETag", testETag) + .withBody(testContent))); + + URL presignedUrl = createPresignedUrl(wireMockRuntimeInfo, presignedUrlPath); + PresignedUrlDownloadRequest request = createRequest(presignedUrl); + + CompletableFuture> result = + presignedUrlExtension.getObject(request, AsyncResponseTransformer.toBytes()); + + ResponseBytes response = result.get(); + assertSuccessfulResponse(response, testContent, testETag); + + verify(exactly(2), getRequestedFor(urlEqualTo(presignedUrlPath))); + } + } + + @Nested + class ErrorScenarios { + @Test + void givenMalformedResponse_whenGetObjectCalled_thenCompletableFutureCompletesExceptionally(WireMockRuntimeInfo wireMockRuntimeInfo) throws Exception { + String presignedUrlPath = "/malformed-response"; + stubFor(get(urlEqualTo(presignedUrlPath)) + .willReturn(aResponse() + .withFault(Fault.CONNECTION_RESET_BY_PEER))); + + URL presignedUrl = createPresignedUrl(wireMockRuntimeInfo, presignedUrlPath); + PresignedUrlDownloadRequest request = createRequest(presignedUrl); + + CompletableFuture> result = + presignedUrlExtension.getObject(request, AsyncResponseTransformer.toBytes()); + try { + result.get(); + throw new AssertionError("Expected exception was not thrown"); + } catch (ExecutionException e) { + assertThat(e.getCause()).isInstanceOf(SdkClientException.class); + } + } + + @Test + void givenNotFoundError_whenGetObjectCalled_thenCompletableFutureCompletesExceptionally(WireMockRuntimeInfo wireMockRuntimeInfo) { + String urlPath = "/nonexistent-key"; + + stubFor(get(urlEqualTo(urlPath)) + .willReturn(aResponse() + .withStatus(404) + .withBody("NoSuchKey"))); + + URL presignedUrl; + try { + presignedUrl = new URL(wireMockRuntimeInfo.getHttpBaseUrl() + urlPath); + } catch (Exception e) { + throw new RuntimeException(e); + } + + PresignedUrlDownloadRequest request = PresignedUrlDownloadRequest.builder() + .presignedUrl(presignedUrl) + .build(); + + CompletableFuture> result = + presignedUrlExtension.getObject(request, AsyncResponseTransformer.toBytes()); + + assertThatThrownBy(() -> result.get()) + .isInstanceOf(ExecutionException.class); + } + + @Test + void givenAccessDeniedError_whenGetObjectCalled_thenCompletableFutureCompletesExceptionally(WireMockRuntimeInfo wireMockRuntimeInfo) { + String urlPath = "/forbidden-key"; + + stubFor(get(urlEqualTo(urlPath)) + .willReturn(aResponse() + .withStatus(403) + .withBody("AccessDenied"))); + + URL presignedUrl; + try { + presignedUrl = new URL(wireMockRuntimeInfo.getHttpBaseUrl() + urlPath); + } catch (Exception e) { + throw new RuntimeException(e); + } + + PresignedUrlDownloadRequest request = PresignedUrlDownloadRequest.builder() + .presignedUrl(presignedUrl) + .build(); + + CompletableFuture> result = + presignedUrlExtension.getObject(request, AsyncResponseTransformer.toBytes()); + + assertThatThrownBy(() -> result.get()) + .isInstanceOf(ExecutionException.class); + } + } + + @Nested + class ConcurrentOperations { + @Test + void givenMultipleRequests_whenGetObjectCalledConcurrently_thenAllDownloadsComplete(WireMockRuntimeInfo wireMockRuntimeInfo) throws Exception { + int concurrentRequests = 5; + List contents = new ArrayList<>(); + List eTags = new ArrayList<>(); + List paths = new ArrayList<>(); + + for (int i = 0; i < concurrentRequests; i++) { + String content = "Content for concurrent download " + i; + String eTag = "\"concurrent-etag-" + i + "\""; + String path = "/concurrent-" + i; + + contents.add(content); + eTags.add(eTag); + paths.add(path); + + stubSuccessResponse(path, content, eTag); + } + + List>> futures = new ArrayList<>(); + for (int i = 0; i < concurrentRequests; i++) { + URL presignedUrl = createPresignedUrl(wireMockRuntimeInfo, paths.get(i)); + PresignedUrlDownloadRequest request = createRequest(presignedUrl); + + futures.add(presignedUrlExtension.getObject(request, AsyncResponseTransformer.toBytes())); + } + + CompletableFuture allFutures = CompletableFuture.allOf(futures.toArray(new CompletableFuture[0])); + allFutures.get(); + + for (int i = 0; i < concurrentRequests; i++) { + ResponseBytes response = futures.get(i).get(); + assertSuccessfulResponse(response, contents.get(i), eTags.get(i)); + } + } + } + + // Helper methods + private URL createPresignedUrl(WireMockRuntimeInfo wireMockRuntimeInfo, String path) { + try { + return new URL(wireMockRuntimeInfo.getHttpBaseUrl() + path); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + private PresignedUrlDownloadRequest createRequest(URL presignedUrl) { + return PresignedUrlDownloadRequest.builder() + .presignedUrl(presignedUrl) + .build(); + } + + private String generateRandomETag() { + return "\"" + UUID.randomUUID().toString().replace("-", "") + "\""; + } + + private void stubSuccessResponse(String path, String content, String eTag) { + stubFor(get(urlEqualTo(path)) + .willReturn(aResponse() + .withStatus(200) + .withHeader("ETag", eTag) + .withBody(content))); + } + + private void stubErrorResponse(String path, int statusCode, String body) { + stubFor(get(urlEqualTo(path)) + .willReturn(aResponse() + .withStatus(statusCode) + .withBody(body))); + } + + private void assertSuccessfulResponse(ResponseBytes response, String expectedContent, String expectedETag) { + assertThat(response).isNotNull(); + assertThat(response.asUtf8String()).isEqualTo(expectedContent); + assertThat(response.response().eTag()).isEqualTo(expectedETag); + } + +} \ No newline at end of file diff --git a/services/s3/src/test/java/software/amazon/awssdk/services/s3/presignedurl/model/PresignedUrlDownloadRequestTest.java b/services/s3/src/test/java/software/amazon/awssdk/services/s3/presignedurl/model/PresignedUrlDownloadRequestTest.java new file mode 100644 index 000000000000..1cc7b34364f0 --- /dev/null +++ b/services/s3/src/test/java/software/amazon/awssdk/services/s3/presignedurl/model/PresignedUrlDownloadRequestTest.java @@ -0,0 +1,115 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.services.s3.presignedurl.model; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.net.URL; +import nl.jqno.equalsverifier.EqualsVerifier; +import org.junit.jupiter.api.Test; + +class PresignedUrlDownloadRequestTest { + + @Test + void equalsAndHashCode_shouldFollowContract() { + EqualsVerifier.forClass(PresignedUrlDownloadRequest.class) + .verify(); + } + + @Test + void builder_shouldCreateRequestWithAllFields() throws Exception { + URL url = new URL("https://example.com"); + PresignedUrlDownloadRequest request = PresignedUrlDownloadRequest.builder() + .presignedUrl(url) + .range("bytes=0-100") + .build(); + + assertThat(request.presignedUrl()).isEqualTo(url); + assertThat(request.range()).isEqualTo("bytes=0-100"); + } + + @Test + void builder_shouldCreateRequestWithOnlyRequiredFields() throws Exception { + URL url = new URL("https://example.com"); + PresignedUrlDownloadRequest request = PresignedUrlDownloadRequest.builder() + .presignedUrl(url) + .build(); + + assertThat(request.presignedUrl()).isEqualTo(url); + assertThat(request.range()).isNull(); + } + + @Test + void toBuilder_shouldCreateBuilderFromExistingRequest() throws Exception { + URL url = new URL("https://example.com"); + PresignedUrlDownloadRequest original = PresignedUrlDownloadRequest.builder() + .presignedUrl(url) + .range("bytes=0-100") + .build(); + + PresignedUrlDownloadRequest copy = original.toBuilder().build(); + + assertThat(copy.presignedUrl()).isEqualTo(original.presignedUrl()); + assertThat(copy.range()).isEqualTo(original.range()); + } + + @Test + void toBuilder_shouldAllowModification() throws Exception { + URL url1 = new URL("https://example.com"); + URL url2 = new URL("https://other.com"); + PresignedUrlDownloadRequest original = PresignedUrlDownloadRequest.builder() + .presignedUrl(url1) + .range("bytes=0-100") + .build(); + + PresignedUrlDownloadRequest modified = original.toBuilder() + .presignedUrl(url2) + .range("bytes=200-300") + .build(); + + assertThat(modified.presignedUrl()).isEqualTo(url2); + assertThat(modified.range()).isEqualTo("bytes=200-300"); + // Original unchanged + assertThat(original.presignedUrl()).isEqualTo(url1); + assertThat(original.range()).isEqualTo("bytes=0-100"); + } + + @Test + void toString_shouldContainActualFieldValues() throws Exception { + URL url = new URL("https://example.com"); + String range = "bytes=0-100"; + + PresignedUrlDownloadRequest request = PresignedUrlDownloadRequest.builder() + .presignedUrl(url) + .range(range) + .build(); + + String result = request.toString(); + + assertThat(result) + .isNotNull() + .isNotEmpty() + .contains(request.presignedUrl().toString()) + .contains(request.range()); + + } + + @Test + void serializableBuilderClass_shouldReturnCorrectClass() { + assertThat(PresignedUrlDownloadRequest.serializableBuilderClass()) + .isEqualTo(PresignedUrlDownloadRequest.BuilderImpl.class); + } +}