java.util.NoSuchElementException thrown after the HttpClientRequest.reset() is called.

[calebkiage]

Version

vertx-core: 4.5.8

Context

I encountered a NoSuchElementException while testing the PR https://github.com/quarkusio/quarkus/pull/41990 using this project

The exception is thrown after a few attempts at reproducing it with the below stack trace:

stack trace

Uncaught exception received by Vert.x: java.util.NoSuchElementException
	at java.base/java.util.ArrayDeque.removeFirst(ArrayDeque.java:362)
	at java.base/java.util.ArrayDeque.pop(ArrayDeque.java:593)
	at io.vertx.core.http.impl.Http1xClientConnection.endRequest(Http1xClientConnection.java:290)
	at io.vertx.core.http.impl.Http1xClientConnection.beginRequest(Http1xClientConnection.java:251)
	at io.vertx.core.http.impl.Http1xClientConnection.access$800(Http1xClientConnection.java:74)
	at io.vertx.core.http.impl.Http1xClientConnection$StreamImpl.writeHead(Http1xClientConnection.java:541)
	at io.vertx.core.http.impl.Http1xClientConnection$StreamImpl.writeHead(Http1xClientConnection.java:526)
	at io.vertx.core.http.impl.HttpClientRequestImpl.doWrite(HttpClientRequestImpl.java:522)
	at io.vertx.core.http.impl.HttpClientRequestImpl.write(HttpClientRequestImpl.java:492)
	at io.vertx.core.http.impl.HttpClientRequestImpl.end(HttpClientRequestImpl.java:438)
	at io.vertx.core.http.impl.HttpClientRequestImpl.end(HttpClientRequestImpl.java:432)
	at io.vertx.core.http.HttpClientRequest.send(HttpClientRequest.java:400)
	at org.example.CallCancelResource$cancel1$2$1$1$1.handle(ExampleResource.kt:50)
	at org.example.CallCancelResource$cancel1$2$1$1$1.handle(ExampleResource.kt:43)
	at io.vertx.core.impl.future.FutureImpl$4.onSuccess(FutureImpl.java:176)
	at io.vertx.core.impl.future.FutureBase.emitSuccess(FutureBase.java:66)
	at io.vertx.core.impl.future.FutureImpl.tryComplete(FutureImpl.java:259)
	at io.vertx.core.http.impl.HttpClientImpl.lambda$null$3(HttpClientImpl.java:392)
	at io.vertx.core.impl.future.FutureImpl$4.onSuccess(FutureImpl.java:176)
	at io.vertx.core.impl.future.FutureBase.lambda$emitSuccess$0(FutureBase.java:60)
	at io.vertx.core.impl.ContextImpl.execute(ContextImpl.java:298)
	at io.vertx.core.impl.DuplicatedContext.execute(DuplicatedContext.java:169)
	at io.vertx.core.impl.future.FutureBase.emitSuccess(FutureBase.java:57)
	at io.vertx.core.impl.future.FutureImpl.addListener(FutureImpl.java:231)
	at io.vertx.core.impl.future.FutureImpl.onComplete(FutureImpl.java:199)
	at io.vertx.core.http.impl.Http1xClientConnection.createStream(Http1xClientConnection.java:1269)
	at io.vertx.core.http.impl.HttpClientImpl.lambda$doRequest$4(HttpClientImpl.java:372)
	at io.vertx.core.net.impl.pool.Endpoint.lambda$getConnection$0(Endpoint.java:52)
	at io.vertx.core.http.impl.SharedClientHttpStreamEndpoint$Request.handle(SharedClientHttpStreamEndpoint.java:162)
	at io.vertx.core.http.impl.SharedClientHttpStreamEndpoint$Request.handle(SharedClientHttpStreamEndpoint.java:123)
	at io.vertx.core.impl.ContextImpl.emit(ContextImpl.java:328)
	at io.vertx.core.impl.ContextImpl.emit(ContextImpl.java:321)
	at io.vertx.core.net.impl.pool.SimpleConnectionPool$LeaseImpl.emit(SimpleConnectionPool.java:714)
	at io.vertx.core.net.impl.pool.SimpleConnectionPool$ConnectSuccess$2.run(SimpleConnectionPool.java:337)
	at io.vertx.core.net.impl.pool.Task.runNextTasks(Task.java:43)
	at io.vertx.core.net.impl.pool.CombinerExecutor.submit(CombinerExecutor.java:91)
	at io.vertx.core.net.impl.pool.SimpleConnectionPool.execute(SimpleConnectionPool.java:244)
	at io.vertx.core.net.impl.pool.SimpleConnectionPool.lambda$connect$2(SimpleConnectionPool.java:256)
	at io.vertx.core.http.impl.SharedClientHttpStreamEndpoint.lambda$connect$2(SharedClientHttpStreamEndpoint.java:102)
	at io.vertx.core.impl.future.FutureImpl$4.onSuccess(FutureImpl.java:176)
	at io.vertx.core.impl.future.FutureBase.emitSuccess(FutureBase.java:66)
	at io.vertx.core.impl.future.FutureImpl.tryComplete(FutureImpl.java:259)
	at io.vertx.core.impl.future.Composition$1.onSuccess(Composition.java:62)
	at io.vertx.core.impl.future.FutureBase.emitSuccess(FutureBase.java:66)
	at io.vertx.core.impl.future.FutureImpl.addListener(FutureImpl.java:231)
	at io.vertx.core.impl.future.Composition.onSuccess(Composition.java:43)
	at io.vertx.core.impl.future.FutureBase.emitSuccess(FutureBase.java:66)
	at io.vertx.core.impl.future.FutureImpl.tryComplete(FutureImpl.java:259)
	at io.vertx.core.Promise.complete(Promise.java:66)
	at io.vertx.core.net.impl.NetClientImpl.lambda$connected$9(NetClientImpl.java:343)
	at io.vertx.core.net.impl.VertxHandler.setConnection(VertxHandler.java:82)
	at io.vertx.core.net.impl.VertxHandler.handlerAdded(VertxHandler.java:88)
	at io.netty.channel.AbstractChannelHandlerContext.callHandlerAdded(AbstractChannelHandlerContext.java:1130)
	at io.netty.channel.DefaultChannelPipeline.callHandlerAdded0(DefaultChannelPipeline.java:608)
	at io.netty.channel.DefaultChannelPipeline.addLast(DefaultChannelPipeline.java:222)
	at io.netty.channel.DefaultChannelPipeline.addLast(DefaultChannelPipeline.java:194)
	at io.vertx.core.net.impl.NetClientImpl.connected(NetClientImpl.java:345)
	at io.vertx.core.net.impl.NetClientImpl.lambda$connectInternal2$3(NetClientImpl.java:307)
	at io.vertx.core.impl.ContextInternal.dispatch(ContextInternal.java:279)
	at io.vertx.core.net.impl.ChannelProvider.connected(ChannelProvider.java:172)
	at io.vertx.core.net.impl.ChannelProvider.lambda$handleConnect$0(ChannelProvider.java:155)
	at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
	at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
	at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
	at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
	at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
	at io.netty.util.concurrent.DefaultPromise.setSuccess0(DefaultPromise.java:625)
	at io.netty.util.concurrent.DefaultPromise.trySuccess(DefaultPromise.java:105)
	at io.netty.channel.DefaultChannelPromise.trySuccess(DefaultChannelPromise.java:84)
	at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:305)
	at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:340)
	at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
	at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
	at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
	at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
	at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	at java.base/java.lang.Thread.run(Thread.java:840)

Just before the exception, the error message below is logged:

ERROR [io.ver.cor.htt.imp.HttpClientRequestImpl] (vert.x-eventloop-thread-1) Stream reset: 0

Do you have a reproducer?

https://github.com/calebkiage/quarkus-kotlin-cancellation-test

Steps to reproduce

1. Run the project

   `./gradlew quarkusDev`
   

2. Call the endpoint http://localhost:9000/call/cancel1 with a short timeout until you see the message Stream reset: 0 followed by the exception:

   curl -X GET --location "http://localhost:9000/call/cancel1" -H "Content-Type: */*" -H "X-TIMEOUT: 1"
   

Extra

OS: Windows 11 OS Version: 23H2 Java Version: openjdk version "17.0.11" 2024-04-16 Kotlin Version: 2.0.0

[calebkiage]

I believe the error is caused by: https://github.com/eclipse-vertx/vert.x/blob/2b0ce5a3966e0bd4ad4cec863cf8b673a93c8599/src/main/java/io/vertx/core/http/impl/Http1xClientConnection.java#L313-L324

The stream is removed from the requests deque on resetting if inflight is false, but then the possibility of an empty request queue isn't checked when calling endRequests(). In this case, reset() was called before send()

Here's a sequence of calls:

1. io.vertx.core.http.impl.Http1xClientConnection#createStream called which pushes a new stream to requests.
2. io.vertx.core.http.HttpClientRequest#reset() called and pops the just added stream.
3. io.vertx.core.http.HttpClientRequest#send() called.
4. send() eventually calls io.vertx.core.http.impl.Http1xClientConnection#endRequest(Stream s) which tries to call requests.pop() on an empty stream and fails.

I think any call that acts on the stream should short circuit if the requests deque is empty. I can contribute a fix.

[vietj]

thank you it looks like a bug

[tsegismont]

Fixed by #5411

[demonbug]

i got a similar error on version 4.5.13

- [AbstractEventExecutor.java:safeExecute:168] - [] - A task raised an exception. Task: io.vertx.core.http.impl.Http1xClientConnection$StreamImpl$$Lambda/0x00007fc9d54a8cd8@4509b173
java.util.NoSuchElementException: null
	at java.util.ArrayDeque.removeFirst(Unknown Source) ~[?:?]
	at java.util.ArrayDeque.pop(Unknown Source) ~[?:?]
	at io.vertx.core.http.impl.Http1xClientConnection.endRequest(Http1xClientConnection.java:290) ~[vertx-core-4.5.13.jar!/:4.5.13]
	at io.vertx.core.http.impl.Http1xClientConnection.beginRequest(Http1xClientConnection.java:251) ~[vertx-core-4.5.13.jar!/:4.5.13]
	at io.vertx.core.http.impl.Http1xClientConnection.access$800(Http1xClientConnection.java:74) ~[vertx-core-4.5.13.jar!/:4.5.13]
	at io.vertx.core.http.impl.Http1xClientConnection$StreamImpl.lambda$writeHead$1(Http1xClientConnection.java:550) ~[vertx-core-4.5.13.jar!/:4.5.13]
	at io.vertx.core.http.impl.Http1xClientConnection$StreamImpl.lambda$queueForWrite$4(Http1xClientConnection.java:627) ~[vertx-core-4.5.13.jar!/:4.5.13]
	at io.netty.util.concurrent.AbstractEventExecutor.runTask(AbstractEventExecutor.java:173) ~[netty-common-4.1.118.Final.jar!/:4.1.118.Final]
	at io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:166) ~[netty-common-4.1.118.Final.jar!/:4.1.118.Final]
	at io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:472) ~[netty-common-4.1.118.Final.jar!/:4.1.118.Final]
	at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:566) ~[netty-transport-4.1.118.Final.jar!/:4.1.118.Final]
	at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:998) ~[netty-common-4.1.118.Final.jar!/:4.1.118.Final]
	at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.118.Final.jar!/:4.1.118.Final]
	at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[netty-common-4.1.118.Final.jar!/:4.1.118.Final]
	at java.lang.Thread.run(Unknown Source) [?:?]

[vietj]

@demonbug can you share more details ? do you have a reproducer for it ?

[demonbug]

@demonbug can you share more details ? do you have a reproducer for it ?

sorry i can't reproduce it, the issue occurs in our production environment with a low probability. it seems to be accompanied by server timeouts and client-initiated disconnections.

Env: x86_64 linux 5.10.0-182 openjdk 21.0.7