aws-sdk-cpp
aws-sdk-cpp copied to clipboard
aws
s3-ctr: crash in S3CrtClient when initialization fails
Describe the bug
backtrace:
(gdb) bt
#0 aws_s3_client_make_meta_request (client=0x0, options=options@entry=0x7ffd59c43d90) at ./obj-x86_64-linux-gnu/_deps/sdk-src/crt/aws-crt-cpp/crt/aws-c-s3/source/s3_client.c:651
#1 0x0000562afad89120 in Aws::S3Crt::S3CrtClient::PutObjectAsync(Aws::S3Crt::Model::PutObjectRequest const&, std::function<void (Aws::S3Crt::S3CrtClient const*, Aws::S3Crt::Model::PutObjectRequest const&, Aws::Utils::Outcome<Aws::S3Crt::Model::PutObjectResult, Aws::S3Crt::S3CrtError> const&, std::shared_ptr<Aws::Client::AsyncCallerContext const> const&)> const&, std::shared_ptr<Aws::Client::AsyncCallerContext const> const&) const (this=0x562afc970020,
request=..., handler=..., context=std::shared_ptr<const Aws::Client::AsyncCallerContext> (empty) = {...}) at ./obj-x86_64-linux-gnu/_deps/sdk-src/aws-cpp-sdk-s3-crt/source/S3CrtClient.cpp:574
#2 0x0000562afac88849 in S3Client::uploadFiles (this=this@entry=0x7ffd59c454f0, filesToUpload=std::vector of length 2, capacity 2 = {...}) at ./src/s3_client.cc:270
some data:
(gdb) f 2
#2 0x0000562afac88849 in S3Client::uploadFiles (this=this@entry=0x7ffd59c454f0, filesToUpload=std::vector of length 2, capacity 2 = {...}) at ./src/s3_client.cc:270
(gdb) p *this
$1 = {executor = std::shared_ptr<Aws::Utils::Threading::PooledThreadExecutor> (use count 2, weak count 0) = {get() = 0x562afc8e4c80}, s3Client = std::shared_ptr<Aws::S3Crt::S3CrtClient> (use count 1, weak count 0) = {
get() = 0x562afc970020}, bucketName = "staging", threadCount = 25}
(gdb) p *(Aws::S3Crt::S3CrtClient*)0x562afc970020
$2 = {<Aws::Client::AWSXMLClient> = {<Aws::Client::AWSClient> = {_vptr.AWSClient = 0x562afb180568 <vtable for Aws::S3Crt::S3CrtClient+16>, m_region = "us-east-1",
m_httpClient = std::shared_ptr<Aws::Http::HttpClient> (use count 1, weak count 0) = {get() = 0x562afcaf3ea0}, m_signerProvider = std::shared_ptr<Aws::Auth::AWSAuthSignerProvider> (use count 1, weak count 0) = {
get() = 0x562afc89c890}, m_errorMarshaller = std::shared_ptr<Aws::Client::AWSErrorMarshaller> (use count 1, weak count 0) = {get() = 0x562afc970340},
m_retryStrategy = std::shared_ptr<Aws::Client::RetryStrategy> (use count 1, weak count 0) = {get() = 0x562afc830b60}, m_writeRateLimiter = std::shared_ptr<Aws::Utils::RateLimits::RateLimiterInterface> (empty) = {
get() = 0x0}, m_readRateLimiter = std::shared_ptr<Aws::Utils::RateLimits::RateLimiterInterface> (empty) = {get() = 0x0}, m_userAgent = "aws-sdk-cpp/1.9.338 Linux/5.4.0-104-generic x86_64 GCC/10.2.1",
m_customizedUserAgent = false, m_hash = std::shared_ptr<Aws::Utils::Crypto::Hash> (use count 1, weak count 0) = {get() = 0x562afc884040}, m_requestTimeoutMs = 3000, m_enableClockSkewAdjustment = true,
m_serviceName = "S3"}, <No data fields>}, m_baseUri = "tt4-s3s1-rgw.<domain>", m_scheme = "http", m_enableHostPrefixInjection = false, m_configScheme = "http",
m_executor = std::shared_ptr<Aws::Utils::Threading::Executor> (use count 2, weak count 0) = {get() = 0x562afc8e4c80}, m_s3CrtClient = 0x0, m_s3CrtSigningConfig = {config_type = AWS_SIGNING_CONFIG_AWS,
algorithm = AWS_SIGNING_ALGORITHM_V4, signature_type = AWS_ST_HTTP_REQUEST_HEADERS, region = {len = 9, ptr = 0x7ffd59c44f50 ""}, service = {len = 2, ptr = 0x562afb057822 "s3"}, date = {timestamp = 0, milliseconds = 0,
tz = "\000\000\000\000\000", gmt_time = {tm_sec = 0, tm_min = 0, tm_hour = 0, tm_mday = 0, tm_mon = 0, tm_year = 0, tm_wday = 0, tm_yday = 0, tm_isdst = 0, tm_gmtoff = 0, tm_zone = 0x0}, local_time = {tm_sec = 0, tm_min = 0,
tm_hour = 0, tm_mday = 0, tm_mon = 0, tm_year = 0, tm_wday = 0, tm_yday = 0, tm_isdst = 0, tm_gmtoff = 0, tm_zone = 0x0}, utc_assumed = false}, should_sign_header = 0x0, should_sign_header_ud = 0x0, flags = {
use_double_uri_encode = 0, should_normalize_uri_path = 0, omit_session_token = 0}, signed_body_value = {len = 16, ptr = 0x562afb04b5ea "UNSIGNED-PAYLOAD"}, signed_body_header = AWS_SBHT_X_AMZ_CONTENT_SHA256,
credentials = 0x0, credentials_provider = 0x562afc88c340, expiration_in_seconds = 0}, m_wrappedData = {data = 0x0,
fn = {<std::_Maybe_unary_or_binary_function<void, void*>> = {<std::unary_function<void*, void>> = {<No data fields>}, <No data fields>}, <std::_Function_base> = {static _M_max_size = 16, static _M_max_align = 8, _M_functor = {
_M_unused = {_M_object = 0x562afc88ebd0, _M_const_object = 0x562afc88ebd0, _M_function_pointer = 0x562afc88ebd0,
_M_member_pointer = (void (std::_Undefined_class::*)(std::_Undefined_class * const)) 0x562afc88ebd0, this adjustment 2}, _M_pod_data = "\320\353\210\374*V\000\000\002\000\000\000\000\000\000"}, _M_manager = 0x0},
_M_invoker = 0x562afaca0930 <std::_Function_handler<std::shared_ptr<Aws::Crt::Auth::Credentials>(), std::_Bind<Aws::S3Crt::S3CrtClient::init(const Aws::S3Crt::ClientConfiguration&, std::shared_ptr<Aws::Auth::AWSCredentialsProvider>)::<lambda(const std::shared_ptr<Aws::Auth::AWSCredentialsProvider>&)>(std::shared_ptr<Aws::Auth::AWSCredentialsProvider>)> >::_M_invoke(const std::_Any_data &)>},
clientShutdownSem = std::shared_ptr<Aws::Utils::Threading::Semaphore> (use count 2, weak count 0) = {get() = 0x562afc921c60}},
m_clientShutdownSem = std::shared_ptr<Aws::Utils::Threading::Semaphore> (use count 2, weak count 0) = {get() = 0x562afc921c60}, m_userAgent = "",
m_credProvider = std::shared_ptr<Aws::Auth::AWSCredentialsProvider> (use count 2, weak count 0) = {get() = 0x562afc860b30}, m_crtCredProvider = warning: RTTI symbol not found for class 'std::_Sp_counted_deleter<Aws::Crt::Auth::CredentialsProvider*, Aws::Crt::MakeShared<Aws::Crt::Auth::CredentialsProvider, aws_credentials_provider*&, aws_allocator*&>(aws_allocator*, aws_credentials_provider*&, aws_allocator*&)::{lambda(Aws::Crt::Auth::CredentialsProvider*)#1}, std::allocator<void>, (__gnu_cxx::_Lock_policy)2>'
warning: RTTI symbol not found for class 'std::_Sp_counted_deleter<Aws::Crt::Auth::CredentialsProvider*, Aws::Crt::MakeShared<Aws::Crt::Auth::CredentialsProvider, aws_credentials_provider*&, aws_allocator*&>(aws_allocator*, aws_credentials_provider*&, aws_allocator*&)::{lambda(Aws::Crt::Auth::CredentialsProvider*)#1}, std::allocator<void>, (__gnu_cxx::_Lock_policy)2>'
std::shared_ptr<Aws::Crt::Auth::ICredentialsProvider> (use count 1, weak count 1) = {
get() = 0x562afc88eba0}, m_useVirtualAddressing = false, m_useDualStack = false, m_useArnRegion = false, m_disableMultiRegionAccessPoints = false, m_useCustomEndpoint = true,
m_USEast1RegionalEndpointOption = Aws::S3Crt::US_EAST_1_REGIONAL_ENDPOINT_OPTION::REGIONAL}
The root of this crash must be, that it fails to allocate member m_s3CrtClient. See m_s3CrtClient = 0x0 from printed instance of Aws::S3Crt::S3CrtClient.
void S3CrtClient::init(...)
{
...
if (!m_s3CrtClient)
{
AWS_LOGSTREAM_FATAL(ALLOCATION_TAG, "Failed to allocate aws_s3_client instance, abort.");
}
}
Expected Behavior
I woud expect exception std::bad_aloc in time of invocing.
Aws::MakeShared<Aws::S3Crt::S3CrtClient>(
"s3-client",
makeAwsCredentials(accessKey, secretKey),
makeAwsClientConfiguration(endpoint, executor, partSizeMiB_),
Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Never,
false /* useVirtualAddressing */,
Aws::S3Crt::US_EAST_1_REGIONAL_ENDPOINT_OPTION::NOT_SET)
Current Behavior
Crash when you don't know the root of the problem. It could be anything. For instance the problem in application code.
Second issue I see, as a user of API I cannot validate if is instance of s3 client valid.
Reproduction Steps
N/A
Possible Solution
void S3CrtClient::init(...)
{
...
if (!m_s3CrtClient)
{
AWS_LOGSTREAM_FATAL(ALLOCATION_TAG, "Failed to allocate aws_s3_client instance, abort.");
throw std::bad_alloc();
}
}
I am able to end an application correctly without generating core dump and log this unexpected exception.
but this fails with error error: exception handling disabled, use '-fexceptions' to enable.
So, I cannot say how to fix it.
- checking the member
m_s3CrtClientin all public function is bad. The application will continue running but any operation fails. std::terminate();is also terrible. It will generate core dump. No way the application end correctly. For instance you hold an handle to local instance of sql-lite and your DB is going to be corrupted.- function isValid() - better than nothing
I can fix it, but rather hear some suggestion from maintainers what approach to get.
Additional Information/Context
No response
AWS CPP SDK version used
1.9.338
Compiler and Version used
gcc (Debian 10.2.1-6) 10.2.1 20210110
Operating System and version
Linux
Can you make sure you are using the sdk calls in the correct way? It should look like this:
#include <aws/core/Aws.h>
int main(int argc, char** argv)
{
Aws::SDKOptions options;
Aws::InitAPI(options);
{
// make your SDK calls here.
}
Aws::ShutdownAPI(options);
return 0;
}
The sdk calls need to be made between InitAPI and ShutdownAPI. If you are calling std::terminate in the middle of that it won't allow the sdk to properly clean up it's resources. If you haven't already you can take a look at our s3crt sample here.
Can you make sure you are using the sdk calls in the correct way? It should look like this:
The sdk calls need to be made between
InitAPIandShutdownAPI.
The application is correct. I call InitAPI and ShutdownAPI and not using std::terminate(). The root of the problem is really simple. The mallock() fails. If you think, it is rare, you are right. It didn't happen since I reported it.
It happens only if application runs out resources. But what worries me, how that S3CrtClient handles this problem.
My application would catch the exception, log the situation and terminate it self with good manner. Now it generates coredump.
my app:
auto s3Client = Aws::MakeShared<Aws::S3Crt::S3CrtClient>(..);
and what fails is this line in contructor in S3CrtClient::init
m_s3CrtClient = aws_s3_client_new(Aws::get_aws_allocator(), &s3CrtConfig);
the m_s3CrtClient is NULL and sitiation is only logged:
if (!m_s3CrtClient)
{
AWS_LOGSTREAM_FATAL(ALLOCATION_TAG, "Failed to allocate aws_s3_client instance, abort.");
m_isInitialized = false;
}
so applicaion follows:
s3Client->PutObjectAsync(request, handler);
... // and here it crash when application waiting on result
The issue is API. How I can recognize the initialization of client fails?
I would expect I can do something like:
auto s3Client = Aws::MakeShared<Aws::S3Crt::S3CrtClient>(..);
if (! s3Client->isValid()) {
throw std::runtime_error{"Failed to create s3 client"};
}
s3Client->PutObjectAsync(request, handler);
Now is not possible to know the object is in invalid state and cannot be safely used. I didn't want to suggest how to deal with that. I would prefer the extra function. But it also perfekty ok, that any call on that object simply fails with error that initialization failed. It would require to test m_isInitialized in every public function of S3CrtClient.
@jmklix
Can you update to the latest version of this sdk if you haven't already? A check for if it's initialized was added two months ago. That should fix your problem for you. Please let me know if you have any more questions.
Greetings! It looks like this issue hasn’t been active in longer than a week. We encourage you to check if this is still an issue in the latest release. Because it has been longer than a week since the last update on this, and in the absence of more information, we will be closing this issue soon. If you find that this is still a problem, please feel free to provide a comment or add an upvote to prevent automatic closure, or if the issue is already closed, please feel free to open a new one.
![github-actions[bot] avatar](https://avatars.githubusercontent.com/in/15368?v=4 "Published by a pub.dev verified publisher"){kind=small}
@jmklix It works well only for synced methods. I tested it with version 1.11.151. the synced method have guard macro AWS_OPERATION_GUARD, but this macro is not suitable for async methods, so is not used there.
Here is a program to upload empty file using sync and async method.
{
// cedentials and hostname
std::cout << "started" << std::endl;
setenv("AWS_EC2_METADATA_DISABLED", "true", 1);
Aws::SDKOptions options;
Aws::InitAPI(options);
struct MyS3CrtClient : public Aws::S3Crt::S3CrtClient {
MyS3CrtClient(
const Aws::Auth::AWSCredentials& credentials,
const Aws::S3Crt::ClientConfiguration& clientConfiguration)
: Aws::S3Crt::S3CrtClient{
credentials,
clientConfiguration,
Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Never,
false /* useVirtualAddressing */,
Aws::S3Crt::US_EAST_1_REGIONAL_ENDPOINT_OPTION::NOT_SET}
{}
// to simulate when initialization fails
void initFail() { m_isInitialized = false; }
};
{
std::cout << ">> s3-ctr client started" << std::endl;
auto executor =
Aws::MakeShared<Aws::Utils::Threading::PooledThreadExecutor>("s3b-executor", 5);
Aws::S3Crt::ClientConfiguration configClient;
configClient.endpointOverride = hostname;
configClient.scheme = Aws::Http::Scheme::HTTPS;
configClient.useVirtualAddressing = false;
configClient.enableHostPrefixInjection = false;
configClient.enableEndpointDiscovery = false;
configClient.disableIMDS = true;
configClient.executor = executor;
auto client = MyS3CrtClient{credentials, configClient};
client.initFail();
const auto makeRequest = []() {
Aws::S3Crt::Model::PutObjectRequest request;
request.SetBucket("dummy-bucket");
request.SetKey("object-key");
auto bodyStream = Aws::MakeShared<Aws::FStream>(
"s3-file", "/dev/null", std::ios_base::in | std::ios_base::binary);
if (!bodyStream->good()) {
std::cout << "Failed to open file." << std::endl;
throw std::runtime_error{"Failed to open file."};
}
request.SetBody(bodyStream);
return request;
};
// sync
std::cout << "sync: ";
{
auto request = makeRequest();
Aws::S3Crt::Model::PutObjectOutcome outcome = client.PutObject(request);
std::cout << (outcome.IsSuccess() ? "ok" : outcome.GetError().GetMessage().c_str())
<< std::endl;
}
// async
std::cout << "async: ";
{
auto request = makeRequest();
std::mutex mutex;
std::condition_variable cv;
Aws::S3Crt::PutObjectResponseReceivedHandler handler =
[&](const Aws::S3Crt::S3CrtClient*,
const Aws::S3Crt::Model::PutObjectRequest&,
const Aws::S3Crt::Model::PutObjectOutcome& outcome,
const std::shared_ptr<const Aws::Client::AsyncCallerContext>&) {
auto lock = std::unique_lock<std::mutex>{mutex};
cv.notify_all();
std::cout
<< (outcome.IsSuccess() ? "ok" : outcome.GetError().GetMessage().c_str())
<< std::endl;
};
auto lock = std::unique_lock<std::mutex>{mutex};
client.PutObjectAsync(request, handler);
cv.wait(lock);
}
std::cout << ">> s3-ctr client ended" << std::endl;
}
Aws::ShutdownAPI(options);
std::cout << "ended" << std::endl;
return 0;
}
output:
started
>> s3-ctr client started
sync: Client is not initialized or already terminated
async: ok
>> s3-ctr client ended
ended
both method should check member m_isInitialized and fails.