UploadPart has random NETWORK_CONNECTION error when upload memory size with 10MB or larger

[lleepq]

Describe the bug

UploadPart randomly times out for a minute with error NETWORK_CONNECTION when we use upload buffer size 10MB.

We are using c++ sdk 1.11.60 and we are trying to do multipart uploads in an EC2 instance. Each part we upload is a piece of memory derived from std::streambuff

struct membuf : std::streambuf
{
   membuf(char* begin, char* end) : begin(begin), end(end)
   {
      this->setp(begin, end);
      this->setg(begin, begin, end);
   }

   virtual pos_type seekoff(off_type off, std::ios_base::seekdir dir, std::ios_base::openmode /*which = std::ios_base::in*/) override
   {
      if (dir == std::ios_base::cur)
         gbump(static_cast<int>(off));
      else if (dir == std::ios_base::end)
         setg(begin, end + off, end);
      else if (dir == std::ios_base::beg)
         setg(begin, begin + off, end);

      return gptr() - eback();
   }

   char* begin, * end;
};

then we use it to create Aws::IoStream

   membuf sbuf(partToUpload->partMemory->data(), partToUpload->partMemory->data() + partToUpload->fileSize);

   std::shared_ptr<Aws::IOStream> input_data = Aws::MakeShared<Aws::IOStream>("", &sbuf);

and set it to the upload request.

 uploadRequest.SetBody(input_data); 

It seems to be working better if the memory size is 5MB, but 10MB buffer size will randomly timeout for 1 min, and retrying will fail again with another 1 min timeout again. Since we need to create files larger than 50GB we need buffer size larger than 10MB or more to work.

Please tell us how to upload multipart files with memory buffer size 10MB or more.

We have opened a case with AWS support (case ID 13250834831) but they urged us that we should add a new issue here.

Expected Behavior

No random errors or timeout when calling UploadPart with 10 MB memory buffer size.

Current Behavior

I have the call stack, the error and the log with Trace level up to the poin the error happened. aws_sdk_2023-07-12-14.log

Reproduction Steps

The error is random. I put this app in a loop to make it happen. Just a few iteration should repro in an EC2 instant.

#include <aws/s3/model/PutObjectRequest.h>
#include <aws/s3/model/ListObjectsRequest.h>
#include <aws/s3/model/DeleteObjectRequest.h>
#include <aws/s3/model/CreateMultipartUploadRequest.h>
#include <aws/s3/model/CreateMultipartUploadResult.h>
#include <aws/s3/model/UploadPartRequest.h>
#include <aws/s3/model/UploadPartResult.h>
#include <aws/s3/model/AbortMultipartUploadRequest.h>
#include <aws/s3/model/AbortMultipartUploadResult.h>
#include <aws/s3/model/CompleteMultipartUploadRequest.h>
#include <aws/s3/model/CompleteMultipartUploadResult.h>
#include <aws/s3/model/CompletedPart.h>
#include <aws/s3/model/GetObjectRequest.h>
#include <aws/s3/model/GetObjectResult.h>
#include <aws/s3/model/HeadObjectResult.h>
#include <aws/s3/model/HeadObjectRequest.h>
#include <aws/s3/model/ListPartsResult.h>
#include <aws/s3/model/ListPartsRequest.h>
#include <aws/s3/model/GetObjectAttributesRequest.h>
#include <aws/s3/model/DeleteObjectRequest.h>
#include <aws/s3/model/DeleteObjectResult.h>

#include <aws/core/Aws.h>
#include <aws/s3/S3Client.h>
#include <aws/core/client/ClientConfiguration.h>
#include <fstream>
#include <atomic>



struct membuf : std::streambuf
{
   membuf(char* begin, char* end) : begin(begin), end(end)
   {
      this->setp(begin, end);
      this->setg(begin, begin, end);
   }

   virtual pos_type seekoff(off_type off, std::ios_base::seekdir dir, std::ios_base::openmode /*which = std::ios_base::in*/) override
   {
      if (dir == std::ios_base::cur)
         gbump(static_cast<int>(off));
      else if (dir == std::ios_base::end)
         setg(begin, end + off, end);
      else if (dir == std::ios_base::beg)
         setg(begin, begin + off, end);

      return gptr() - eback();
   }

   char* begin, * end;
};



int main()
{
   Aws::SDKOptions awsOptions;
   awsOptions.loggingOptions.logLevel = Aws::Utils::Logging::LogLevel::Debug;
   Aws::InitAPI(awsOptions);
   std::unique_ptr<Aws::S3::S3Client> s3_client;
   Aws::Client::ClientConfiguration config;
   s3_client.reset(new Aws::S3::S3Client(config));


   Aws::S3::Model::CreateMultipartUploadRequest request;
   Aws::S3::Model::CreateMultipartUploadOutcome requestOutcome;
   Aws::S3::Model::CreateMultipartUploadResult createMUResult;
   const std::string& bucketName = "bmg-xteam-test-bucket";
   const std::string& fileName = "Capture/multipartFile";

   request.WithBucket(bucketName);
   request.WithKey(fileName);
   Aws::String uploadId;

   requestOutcome = s3_client->CreateMultipartUpload(request);

   if (!requestOutcome.IsSuccess())
   {
      auto err = requestOutcome.GetError();
      assert(0);
   }
   else
   {
      createMUResult = requestOutcome.GetResult();
      uploadId = createMUResult.GetUploadId();
   }



   uint32_t partSize = 10 * 1024 * 1024;
   std::unique_ptr<std::vector<char>> memoryForParts;
   memoryForParts.reset(new std::vector<char>(partSize));

   membuf sbuf(memoryForParts->data(), memoryForParts->data() + partSize);
   std::shared_ptr<Aws::IOStream> input_data2 = Aws::MakeShared<Aws::IOStream>("", &sbuf);
   Aws::String etag;
   std::map<uint32_t, Aws::S3::Model::CompletedPart>  comUpload;

   for (auto i = 1; i < 1000; i++)
   {

      // Create the request to upload a part.
      Aws::S3::Model::UploadPartRequest uploadRequest;
      uploadRequest.SetBucket(bucketName);
      uploadRequest.SetKey(fileName);
      uploadRequest.SetUploadId(uploadId);
      uploadRequest.SetPartNumber(i);
      uploadRequest.SetContentLength(partSize);
      uploadRequest.SetBody(input_data2);

      // Upload the part and add the response's ETag to our list.
      Aws::S3::Model::UploadPartOutcome uploadOutcome = s3_client->UploadPart(uploadRequest);

      if (!uploadOutcome.IsSuccess())
      {
         auto err = requestOutcome.GetError();
         assert(0);
      }
      else
      {
         std::cout << "Successfully upload! part "<< i << std::endl;
         etag = uploadOutcome.GetResult().GetETag();
         Aws::S3::Model::CompletedPart comPart;
         comPart.SetETag(etag);
         comPart.SetPartNumber(i);
         comUpload[i] = comPart;
      }

   }


   if (comUpload.size())
   {
      Aws::Vector<Aws::S3::Model::CompletedPart> awsUploadList;

      for (auto part : comUpload)
      {
         awsUploadList.push_back(part.second);
      }

      Aws::S3::Model::CompletedMultipartUpload comMulpartUpload;

      comMulpartUpload.SetParts(awsUploadList);

      Aws::S3::Model::CompleteMultipartUploadRequest  completeUploadRequest;
      completeUploadRequest.SetBucket(bucketName);
      completeUploadRequest.SetKey(fileName);
      completeUploadRequest.SetUploadId(uploadId);
      completeUploadRequest.SetMultipartUpload(comMulpartUpload);
      //completeUploadRequest.

      Aws::S3::Model::CompleteMultipartUploadOutcome outcome = s3_client->CompleteMultipartUpload(completeUploadRequest);
      if (outcome.IsSuccess())
      {
         std::cout << "Successfully complete multipart upload! " << std::endl;
      }
      else
      {
         std::cout << "Failed to complete multipart upload!!! " << std::endl;
         auto err = outcome.GetError();
         std::cout << "Failed to complete multipart upload!!! " << std::endl;
      }

      comUpload.clear();


   }
}

Possible Solution

No response

Additional Information/Context

No response

AWS CPP SDK version used

1.11.60

Compiler and Version used

Visual Studio 2022

Operating System and version

Windows Server 2019