aws-sdk-java-v2
aws-sdk-java-v2 copied to clipboard
Published
20 hours ago •
aws
aws
Multipart upload failure from crtBuilder() with S3 like Storage ex S3 Cohesity
Describe the bug
I am trying to upload file to S3 using crtBuilder & file size if of approx 12MB
below is S3ClientConfiguration
return S3AsyncClient
.crtBuilder()
/// .builder()
.credentialsProvider(StaticCredentialsProvider.create(credentials))
.forcePathStyle(true)
.region(Region.AF_SOUTH_1)
.endpointOverride(new URI(endpoint))
// .multipartEnabled(true)
.build();
}
and below are two methods one use transferManager to upload and another uses s3Asyncclient Directly .. both of which are failing with this error in log stack java.util.concurrent.CompletionException: software.amazon.awssdk.services.s3.model.S3Exception: null (Service: S3, Status Code: 400, Request ID: null)
Implementation with S3TransferManager
private S3TransferManager createTransferManager() {
return S3TransferManager.builder()
.s3Client(amazonS3Client).build();
}
public void uploadMultiPart(String size) {
final String fileName;
String fileAbsolutePath = "";
if(size != null) {
fileName = "randomBigFile.txt";
fileAbsolutePath = "C:\\DEV\\projects\\demo\\demo\\s3Resources\\randomBigFile.txt";
} else {
fileName = "randomSmallFile.txt";
fileAbsolutePath = "C:\\DEV\\projects\\demo\\demo\\s3Resources\\randomSmallFile.txt";
}
try(S3TransferManager transferManager = createTransferManager()) {
UploadFileRequest uploadFileRequest = UploadFileRequest.builder()
.putObjectRequest(req -> req
.bucket(awsS3Configuration.bucketName)
.key("archived/" + fileName)
.build())
.source(Paths.get(fileAbsolutePath))
.build();
try {
transferManager.uploadFile(uploadFileRequest).completionFuture().join();
System.out.println("Upload completed!");
} catch (Exception e) {
e.printStackTrace();
}
} catch (Exception e) {
log.error("Error while listing files via Storage Provider {}", e.getMessage(), e);
throw new RuntimeException("Error While downloading Objects from Shared Storage", e);
}
}
here is one with S3AsyncClient Directly
public void upload() {
String fileName = "someFile.txt";
String fileAbsolutePath = "C:\\DEV\\projects\\demo\\demo\\s3Resources\\someFile.txt";
try {
PutObjectResponse uploadResponse = amazonS3Client
.putObject(req -> req
.bucket(awsS3Configuration.bucketName)
.key("archived/" + fileName),
Paths.get(fileAbsolutePath)
).join();
log.info("File archived from S3: {}, response = {}", fileName, uploadResponse);
} catch(Exception e) {
log.error("Got Exception while copying Object from S3: {}", fileName, e);
throw new RuntimeException(e);
}
}
Just FYI below are my dependencies
<dependencies>
<dependency>
<groupId>software.amazon.awssdk</groupId>
<artifactId>bom</artifactId>
<type>pom</type>
<scope>import</scope>
<version>2.25.60</version>
</dependency>
</dependencies>
<dependencies>
<dependency>
<groupId>software.amazon.awssdk</groupId>
<artifactId>s3</artifactId>
</dependency>
<dependency>
<groupId>software.amazon.awssdk.crt</groupId>
<artifactId>aws-crt</artifactId>
<version>0.29.20</version>
</dependency>
<dependency>
<groupId>software.amazon.awssdk</groupId>
<artifactId>s3-transfer-manager</artifactId>
</dependency>
<dependency>
<groupId>software.amazon.awssdk</groupId>
<artifactId>apache-client</artifactId>
</dependency>
Expected Behavior
File should be uploaded without any exceptions
Current Behavior
sdk throwing 400 with below stack trace
java.util.concurrent.CompletionException: software.amazon.awssdk.services.s3.model.S3Exception: null (Service: S3, Status Code: 400, Request ID: null)
at software.amazon.awssdk.utils.CompletableFutureUtils.errorAsCompletionException(CompletableFutureUtils.java:65)
at software.amazon.awssdk.core.internal.http.pipeline.stages.AsyncExecutionFailureExceptionReportingStage.lambda$execute$0(AsyncExecutionFailureExceptionReportingStage.java:51)
at java.base/java.util.concurrent.CompletableFuture.uniHandle(CompletableFuture.java:934)
at java.base/java.util.concurrent.CompletableFuture$UniHandle.tryFire(CompletableFuture.java:911)
at java.base/java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510)
at java.base/java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162)
at software.amazon.awssdk.utils.CompletableFutureUtils.lambda$forwardExceptionTo$0(CompletableFutureUtils.java:79)
at java.base/java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863)
at java.base/java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841)
at java.base/java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510)
at java.base/java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162)
at software.amazon.awssdk.core.internal.http.pipeline.stages.AsyncRetryableStage$RetryingExecutor.maybeAttemptExecute(AsyncRetryableStage.java:103)
at software.amazon.awssdk.core.internal.http.pipeline.stages.AsyncRetryableStage$RetryingExecutor.maybeRetryExecute(AsyncRetryableStage.java:184)
at software.amazon.awssdk.core.internal.http.pipeline.stages.AsyncRetryableStage$RetryingExecutor.lambda$attemptExecute$1(AsyncRetryableStage.java:170)
at java.base/java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863)
at java.base/java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841)
at java.base/java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510)
at java.base/java.util.concurrent.CompletableFuture.complete(CompletableFuture.java:2147)
at software.amazon.awssdk.core.internal.http.pipeline.stages.MakeAsyncHttpRequestStage.lambda$execute$0(MakeAsyncHttpRequestStage.java:110)
at java.base/java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863)
at java.base/java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841)
at java.base/java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510)
at java.base/java.util.concurrent.CompletableFuture.complete(CompletableFuture.java:2147)
at software.amazon.awssdk.core.internal.http.pipeline.stages.MakeAsyncHttpRequestStage.completeResponseFuture(MakeAsyncHttpRequestStage.java:253)
at software.amazon.awssdk.core.internal.http.pipeline.stages.MakeAsyncHttpRequestStage.lambda$executeHttpRequest$3(MakeAsyncHttpRequestStage.java:167)
at java.base/java.util.concurrent.CompletableFuture.uniHandle(CompletableFuture.java:934)
at java.base/java.util.concurrent.CompletableFuture$UniHandle.tryFire(CompletableFuture.java:911)
at java.base/java.util.concurrent.CompletableFuture$Completion.run(CompletableFuture.java:482)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
at java.base/java.lang.Thread.run(Thread.java:840)
Caused by: software.amazon.awssdk.services.s3.model.S3Exception: null (Service: S3, Status Code: 400, Request ID: null)
at software.amazon.awssdk.protocols.xml.internal.unmarshall.AwsXmlPredicatedResponseHandler.handleErrorResponse(AwsXmlPredicatedResponseHandler.java:156)
at software.amazon.awssdk.protocols.xml.internal.unmarshall.AwsXmlPredicatedResponseHandler.handleResponse(AwsXmlPredicatedResponseHandler.java:108)
at software.amazon.awssdk.protocols.xml.internal.unmarshall.AwsXmlPredicatedResponseHandler.handle(AwsXmlPredicatedResponseHandler.java:85)
at software.amazon.awssdk.protocols.xml.internal.unmarshall.AwsXmlPredicatedResponseHandler.handle(AwsXmlPredicatedResponseHandler.java:43)
at software.amazon.awssdk.core.internal.handler.BaseClientHandler.lambda$successTransformationResponseHandler$7(BaseClientHandler.java:279)
at software.amazon.awssdk.core.internal.http.async.AsyncResponseHandler.lambda$prepare$0(AsyncResponseHandler.java:92)
at java.base/java.util.concurrent.CompletableFuture$UniCompose.tryFire(CompletableFuture.java:1150)
at java.base/java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510)
at java.base/java.util.concurrent.CompletableFuture.complete(CompletableFuture.java:2147)
at software.amazon.awssdk.core.internal.http.async.AsyncResponseHandler$BaosSubscriber.onComplete(AsyncResponseHandler.java:135)
at software.amazon.awssdk.core.internal.metrics.BytesReadTrackingPublisher$BytesReadTracker.onComplete(BytesReadTrackingPublisher.java:74)
at software.amazon.awssdk.utils.async.SimplePublisher.doProcessQueue(SimplePublisher.java:275)
at software.amazon.awssdk.utils.async.SimplePublisher.processEventQueue(SimplePublisher.java:224)
at software.amazon.awssdk.utils.async.SimplePublisher.complete(SimplePublisher.java:157)
at java.base/java.util.concurrent.CompletableFuture.uniRunNow(CompletableFuture.java:819)
at java.base/java.util.concurrent.CompletableFuture.uniRunStage(CompletableFuture.java:803)
at java.base/java.util.concurrent.CompletableFuture.thenRun(CompletableFuture.java:2195)
at software.amazon.awssdk.services.s3.internal.crt.S3CrtResponseHandlerAdapter.onErrorResponseComplete(S3CrtResponseHandlerAdapter.java:181)
at software.amazon.awssdk.services.s3.internal.crt.S3CrtResponseHandlerAdapter.handleError(S3CrtResponseHandlerAdapter.java:160)
at software.amazon.awssdk.services.s3.internal.crt.S3CrtResponseHandlerAdapter.onFinished(S3CrtResponseHandlerAdapter.java:129)
at software.amazon.awssdk.crt.s3.S3MetaRequestResponseHandlerNativeAdapter.onFinished(S3MetaRequestResponseHandlerNativeAdapter.java:25)
Reproduction Steps
use above provided code with above provided dependencies
Possible Solution
No response
Additional Information/Context
No response
AWS Java SDK version used
2.25.60
JDK version used
java 1.8
Operating System and version
linux
