AsyncHTTPClient Transaction StateMachine Fatal Error with large Response Bodies

[rnantes]

When function try await response.body.collect(upTo: maxBodySize) is called where response is a HTTPClientResponse and maxBodySize = 1048576 (i.e 1 MB) and the response body is greater than 1MB then AsyncHTTPClient/Transaction+StateMachine fatal errors. I would expect that the function to throw but I wouldn't expect the fatal error.

example code:

let logger = Logger.init(label: "my-logger")

var config = HTTPClient.Configuration()
config.timeout =  HTTPClient.Configuration.Timeout.init(connect: .seconds(60), read: .seconds(30))
config.redirectConfiguration = HTTPClient.Configuration.RedirectConfiguration.follow(max: 20, allowCycles: true)
config.httpVersion = .automatic
config.decompression = .enabled(limit: .none)
config.connectionPool = ConnectionPool()

let httpClient = HTTPClient.init(eventLoopGroupProvider: .shared(eventLoopGroup),
                                             configuration: config,
                                             backgroundActivityLogger: logger)


var request = HTTPClientRequest.init(url: url)
request.method = .GET
let response = try await httpClient.execute(request, timeout: HTTPClientRequest.standardTimeout)
if !response.hasSuccessStatusCode() {
    throw SomeError.non200StatusCode
}

// Set a maxBodySize of 1 megabyte
let maxBodySize = 1024 * 1024

// This line  does throw as expected when the body is over 1MB in size but, unexpectedly it causes a fatal error
let body = try await response.body.collect(upTo: maxBodySize)

// This code is never reached
guard let bodyString = body.getString(at: 0, length: body.readableBytes) else {
    throw AsyncHTTPGetHelperError.couldNotReadStringFromResponseBody
}
return bodyString

This is the error I am seeing

2022-08-09T01:37:15-0400 error AsyncHTTPGetHelper : FAILED - NIOTooManyBytesError()
AsyncHTTPClient/Transaction+StateMachine.swift:699: Fatal error: Already received an eof or error before. Must not receive further events. Invalid state: executing(AsyncHTTPClient.Transaction.StateMachine.ExecutionContext(executor: AsyncHTTPClient.HTTP2ClientRequestHandler, allocator: NIOCore.ByteBufferAllocator(malloc: (Function), realloc: (Function), free: (Function), memcpy: (Function)), continuation: Swift.CheckedContinuation<AsyncHTTPClient.HTTPClientResponse, Swift.Error>(canary: Swift.CheckedContinuationCanary)), AsyncHTTPClient.Transaction.StateMachine.(unknown context at $12ba8a92c).RequestStreamState.finished, AsyncHTTPClient.Transaction.StateMachine.(unknown context at $12ba8a9ec).ResponseStreamState.buffering(AsyncHTTPClient.HTTPClientResponse.Body.IteratorStream.ID(objectID: ObjectIdentifier(0x0000600000260540)), [ _ _ _ _ _ _ _ <ByteBuffer { readerIndex: 0, writerIndex: 16384, readableBytes: 16384, capacity: 16384, storageCapacity: 16384, slice: _ByteBufferSlice { 0..<16384 }, storage: 0x000000010311bc00 (16384 bytes) } ByteBuffer { readerIndex: 0, writerIndex: 795, readableBytes: 795, capacity: 16384, storageCapacity: 16384, slice: _ByteBufferSlice { 0..<16384 }, storage: 0x000000010311fc00 (16384 bytes) } ByteBuffer { readerIndex: 0, writerIndex: 16384, readableBytes: 16384, capacity: 16384, storageCapacity: 16384, slice: _ByteBufferSlice { 0..<16384 }, storage: 0x0000000103123c00 (16384 bytes) } ByteBuffer { readerIndex: 0, writerIndex: 16384, readableBytes: 16384, capacity: 16384, storageCapacity: 16384, slice: _ByteBufferSlice { 0..<16384 }, storage: 0x0000000103127c00 (16384 bytes) } ByteBuffer { readerIndex: 0, writerIndex: 16335, readableBytes: 16335, capacity: 16384, storageCapacity: 16384, slice: _ByteBufferSlice { 0..<16384 }, storage: 0x000000010312bc00 (16384 bytes) } ByteBuffer { readerIndex: 0, writerIndex: 16384, readableBytes: 16384, capacity: 16384, storageCapacity: 16384, slice: _ByteBufferSlice { 0..<16384 }, storage: 0x000000010312fc00 (16384 bytes) } ByteBuffer { readerIndex: 0, writerIndex: 13934, readableBytes: 13934, capacity: 16384, storageCapacity: 16384, slice: _ByteBufferSlice { 0..<16384 }, storage: 0x00000001030d3c00 (16384 bytes) } >_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ ] (bufferCapacity: 32, ringLength: 7), next: AsyncHTTPClient.Transaction.StateMachine.(unknown context at $12ba8a9ec).ResponseStreamState.Next.error(HTTPClientError.cancelled)))
2022-08-09 01:37:15.490358-0400 xctest[60505:244935] AsyncHTTPClient/Transaction+StateMachine.swift:699: Fatal error: Already received an eof or error before. Must not receive further events. Invalid state: executing(AsyncHTTPClient.Transaction.StateMachine.ExecutionContext(executor: AsyncHTTPClient.HTTP2ClientRequestHandler, allocator: NIOCore.ByteBufferAllocator(malloc: (Function), realloc: (Function), free: (Function), memcpy: (Function)), continuation: Swift.CheckedContinuation<AsyncHTTPClient.HTTPClientResponse, Swift.Error>(canary: Swift.CheckedContinuationCanary)), AsyncHTTPClient.Transaction.StateMachine.(unknown context at $12ba8a92c).RequestStreamState.finished, AsyncHTTPClient.Transaction.StateMachine.(unknown context at $12ba8a9ec).ResponseStreamState.buffering(AsyncHTTPClient.HTTPClientResponse.Body.IteratorStream.ID(objectID: ObjectIdentifier(0x0000600000260540)), [ _ _ _ _ _ _ _ <ByteBuffer { readerIndex: 0, writerIndex: 16384, readableBytes: 16384, capacity: 16384, storageCapacity: 16384, slice: _ByteBufferSlice { 0..<16384 }, storage: 0x000000010311bc00 (16384 bytes) } ByteBuffer { readerIndex: 0, writerIndex: 795, readableBytes: 795, capacity: 16384, storageCapacity: 16384, slice: _ByteBufferSlice { 0..<16384 }, storage: 0x000000010311fc00 (16384 bytes) } ByteBuffer { readerIndex: 0, writerIndex: 16384, readableBytes: 16384, capacity: 16384, storageCapacity: 16384, slice: _ByteBufferSlice { 0..<16384 }, storage: 0x0000000103123c00 (16384 bytes) } ByteBuffer { readerIndex: 0, writerIndex: 16384, readableBytes: 16384, capacity: 16384, storageCapacity: 16384, slice: _ByteBufferSlice { 0..<16384 }, storage: 0x0000000103127c00 (16384 bytes) } ByteBuffer { readerIndex: 0, writerIndex: 16335, readableBytes: 16335, capacity: 16384, storageCapacity: 16384, slice: _ByteBufferSlice { 0..<16384 }, storage: 0x000000010312bc00 (16384 bytes) } ByteBuffer { readerIndex: 0, writerIndex: 16384, readableBytes: 16384, capacity: 16384, storageCapacity: 16384, slice: _ByteBufferSlice { 0..<16384 }, storage: 0x000000010312fc00 (16384 bytes) } ByteBuffer { readerIndex: 0, writerIndex: 13934, readableBytes: 13934, capacity: 16384, storageCapacity: 16384, slice: _ByteBufferSlice { 0..<16384 }, storage: 0x00000001030d3c00 (16384 bytes) } >_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ ] (bufferCapacity: 32, ringLength: 7), next: AsyncHTTPClient.Transaction.StateMachine.(unknown context at $12ba8a9ec).ResponseStreamState.Next.error(HTTPClientError.cancelled)))
Program ended with exit code: 9

[dnadoba]

Hi Reid, thank you for the bug report! Haven't tried to reproduce it myself but this should definitely not happen! Do you have any URL you could share where this reproduces?

Looking at the code, I think we hit a race condition where we cancel the request because we have received too many bytes but still receive some bytes from the connection afterwards. The fix could be as easy as tolerating more bytes while in the executing but finished state. That being said, I want to take a closer look as I'm suspecting we shouldn't even be in the executing but finished state after the request was canceled.

[rnantes]

Hi David, the test URL I used to produce the fatal error was https://api.weather.gov/zones/forecast/AKZ026/forecast, it returns a body with a size of 6.67 MB.

[dnadoba]

Just for reference a full test case:

let client = HTTPClient(eventLoopGroupProvider: .createNew)

defer { XCTAssertNoThrow(try client.syncShutdown()) }

var request = HTTPClientRequest(url: "http://api.weather.gov/zones/forecast/AKZ026/forecast")
request.headers.add(name: "User-Agent", value: "Swift HTTPClient")
let response = try await client.execute(request, deadline: .now() + .seconds(10))

let maxBodySize = 1024 * 1024

await XCTAssertThrowsError(try await response.body.collect(upTo: maxBodySize)) { error in
    XCTAssert(error is NIOTooManyBytesError)
}

// we need to wait a bit to receive more packets before we shutdown the HTTPClient
try await Task.sleep(nanoseconds: UInt64(TimeAmount.milliseconds(100).nanoseconds))

Note that the URL only responds correctly if a User-Agent header is set, otherwise it responds with 403 forbidden.