akhq
akhq copied to clipboard
tchiotludo
java.lang.OutOfMemoryError while tailing topic with high throughput
Whenever user starts looking at the topic, the records are being tailed. Recently one of the users started to check data from topic which has really high throughput. My assumption is that AKHQ reaches the memory limit and afterwards users are not able to use AKHQ as it throws OutOfMemoryError:
java.lang.OutOfMemoryError: Direct buffer memory at java.base/java.nio.Bits.reserveMemory(Unknown Source) at java.base/java.nio.DirectByteBuffer.
(Unknown Source) at java.base/java.nio.ByteBuffer.allocateDirect(Unknown Source) at io.netty.buffer.PoolArena$DirectArena.allocateDirect(PoolArena.java:649) at io.netty.buffer.PoolArena$DirectArena.newChunk(PoolArena.java:624) at io.netty.buffer.PoolArena.allocateNormal(PoolArena.java:203) at io.netty.buffer.PoolArena.tcacheAllocateSmall(PoolArena.java:173) at io.netty.buffer.PoolArena.allocate(PoolArena.java:134) at io.netty.buffer.PoolArena.reallocate(PoolArena.java:287) at io.netty.buffer.PooledByteBuf.capacity(PooledByteBuf.java:122) at io.netty.buffer.AbstractByteBuf.ensureWritable0(AbstractByteBuf.java:305) at io.netty.buffer.AbstractByteBuf.ensureWritable(AbstractByteBuf.java:280) at io.netty.buffer.AbstractByteBuf.writeBytes(AbstractByteBuf.java:1073) at io.netty.buffer.ByteBufOutputStream.write(ByteBufOutputStream.java:67) at com.fasterxml.jackson.core.json.UTF8JsonGenerator._flushBuffer(UTF8JsonGenerator.java:2171) at com.fasterxml.jackson.core.json.UTF8JsonGenerator.close(UTF8JsonGenerator.java:1214) at com.fasterxml.jackson.databind.ObjectMapper._writeValueAndClose(ObjectMapper.java:4573) at com.fasterxml.jackson.databind.ObjectMapper.writeValue(ObjectMapper.java:3780) at io.micronaut.jackson.databind.JacksonDatabindMapper.writeValue(JacksonDatabindMapper.java:124) at io.micronaut.jackson.databind.JacksonDatabindMapper.writeValue(JacksonDatabindMapper.java:129) at io.micronaut.json.codec.MapperMediaTypeCodec.encode(MapperMediaTypeCodec.java:218) at io.micronaut.json.codec.MapperMediaTypeCodec.encode(MapperMediaTypeCodec.java:268) at io.micronaut.http.server.netty.RoutingInBoundHandler.encodeBodyAsByteBuf(RoutingInBoundHandler.java:1358) at io.micronaut.http.server.netty.RoutingInBoundHandler.encodeBodyWithCodec(RoutingInBoundHandler.java:1299) at io.micronaut.http.server.netty.RoutingInBoundHandler.encodeResponseBody(RoutingInBoundHandler.java:1117) at io.micronaut.http.server.netty.RoutingInBoundHandler.encodeHttpResponse(RoutingInBoundHandler.java:962) at io.micronaut.http.server.netty.RoutingInBoundHandler.access$000(RoutingInBoundHandler.java:144) at io.micronaut.http.server.netty.RoutingInBoundHandler$2.doOnNext(RoutingInBoundHandler.java:579) at io.micronaut.http.server.netty.RoutingInBoundHandler$2.doOnNext(RoutingInBoundHandler.java:571) at io.micronaut.core.async.subscriber.CompletionAwareSubscriber.onNext(CompletionAwareSubscriber.java:52) at reactor.core.publisher.StrictSubscriber.onNext(StrictSubscriber.java:89) at io.micronaut.reactive.reactor.instrument.ReactorSubscriber.onNext(ReactorSubscriber.java:57) at reactor.core.publisher.FluxContextWrite$ContextWriteSubscriber.onNext(FluxContextWrite.java:107) at io.micronaut.reactive.reactor.instrument.ReactorSubscriber.onNext(ReactorSubscriber.java:57) at reactor.core.publisher.FluxOnErrorResume$ResumeSubscriber.onNext(FluxOnErrorResume.java:79) at io.micronaut.reactive.reactor.instrument.ReactorSubscriber.onNext(ReactorSubscriber.java:57) at reactor.core.publisher.FluxFlatMap$FlatMapMain.drainLoop(FluxFlatMap.java:712) at reactor.core.publisher.FluxFlatMap$FlatMapMain.drain(FluxFlatMap.java:588) at reactor.core.publisher.FluxFlatMap$FlatMapInner.onSubscribe(FluxFlatMap.java:955) at io.micronaut.reactive.reactor.instrument.ReactorSubscriber.onSubscribe(ReactorSubscriber.java:50) at reactor.core.publisher.FluxJust.subscribe(FluxJust.java:68) at reactor.core.publisher.Flux.subscribe(Flux.java:8402) at reactor.core.publisher.FluxFlatMap$FlatMapMain.onNext(FluxFlatMap.java:426) at io.micronaut.reactive.reactor.instrument.ReactorSubscriber.onNext(ReactorSubscriber.java:57) at reactor.core.publisher.FluxOnErrorResume$ResumeSubscriber.onNext(FluxOnErrorResume.java:79) at io.micronaut.reactive.reactor.instrument.ReactorSubscriber.onNext(ReactorSubscriber.java:57) at reactor.core.publisher.FluxFlatMap$FlatMapMain.drainLoop(FluxFlatMap.java:712) at reactor.core.publisher.FluxFlatMap$FlatMapMain.drain(FluxFlatMap.java:588) at reactor.core.publisher.FluxFlatMap$FlatMapInner.onSubscribe(FluxFlatMap.java:955) at io.micronaut.reactive.reactor.instrument.ReactorSubscriber.onSubscribe(ReactorSubscriber.java:50) at reactor.core.publisher.FluxJust.subscribe(FluxJust.java:68) at reactor.core.publisher.Flux.subscribe(Flux.java:8402) at reactor.core.publisher.FluxFlatMap$FlatMapMain.onNext(FluxFlatMap.java:426) at io.micronaut.configuration.metrics.binder.web.WebMetricsPublisher$1.onNext(WebMetricsPublisher.java:180) at io.micronaut.configuration.metrics.binder.web.WebMetricsPublisher$1.onNext(WebMetricsPublisher.java:157) at io.micronaut.reactive.reactor.instrument.ReactorSubscriber.onNext(ReactorSubscriber.java:57) at reactor.core.publisher.FluxOnErrorResume$ResumeSubscriber.onNext(FluxOnErrorResume.java:79) at io.micronaut.reactive.reactor.instrument.ReactorSubscriber.onNext(ReactorSubscriber.java:57) at reactor.core.publisher.FluxFlatMap$FlatMapMain.drainLoop(FluxFlatMap.java:712) at reactor.core.publisher.FluxFlatMap$FlatMapMain.drain(FluxFlatMap.java:588) at reactor.core.publisher.FluxFlatMap$FlatMapInner.onSubscribe(FluxFlatMap.java:955) at io.micronaut.reactive.reactor.instrument.ReactorSubscriber.onSubscribe(ReactorSubscriber.java:50) at reactor.core.publisher.FluxJust.subscribe(FluxJust.java:68) at reactor.core.publisher.Flux.subscribe(Flux.java:8402) at reactor.core.publisher.FluxFlatMap$FlatMapMain.onNext(FluxFlatMap.java:426) at io.micronaut.reactive.reactor.instrument.ReactorSubscriber.onNext(ReactorSubscriber.java:57) at reactor.core.publisher.FluxSwitchIfEmpty$SwitchIfEmptySubscriber.onNext(FluxSwitchIfEmpty.java:74) at io.micronaut.reactive.reactor.instrument.ReactorSubscriber.onNext(ReactorSubscriber.java:57) at reactor.core.publisher.Operators$MonoSubscriber.complete(Operators.java:1816) at reactor.core.publisher.MonoFlatMap$FlatMapInner.onNext(MonoFlatMap.java:249) at io.micronaut.reactive.reactor.instrument.ReactorSubscriber.onNext(ReactorSubscriber.java:57) at reactor.core.publisher.MonoNext$NextSubscriber.onNext(MonoNext.java:82) at io.micronaut.reactive.reactor.instrument.ReactorSubscriber.onNext(ReactorSubscriber.java:57) at reactor.core.publisher.FluxSwitchIfEmpty$SwitchIfEmptySubscriber.onNext(FluxSwitchIfEmpty.java:74) at io.micronaut.reactive.reactor.instrument.ReactorSubscriber.onNext(ReactorSubscriber.java:57) at reactor.core.publisher.MonoFlatMapMany$FlatMapManyInner.onNext(MonoFlatMapMany.java:250) at io.micronaut.reactive.reactor.instrument.ReactorSubscriber.onNext(ReactorSubscriber.java:57) at reactor.core.publisher.FluxOnErrorResume$ResumeSubscriber.onNext(FluxOnErrorResume.java:79) at io.micronaut.reactive.reactor.instrument.ReactorSubscriber.onNext(ReactorSubscriber.java:57) at reactor.core.publisher.FluxFlatMap$FlatMapMain.drainLoop(FluxFlatMap.java:712) at reactor.core.publisher.FluxFlatMap$FlatMapMain.drain(FluxFlatMap.java:588) at reactor.core.publisher.FluxFlatMap$FlatMapInner.onSubscribe(FluxFlatMap.java:955) at io.micronaut.reactive.reactor.instrument.ReactorSubscriber.onSubscribe(ReactorSubscriber.java:50) at reactor.core.publisher.FluxJust.subscribe(FluxJust.java:68) at reactor.core.publisher.Flux.subscribe(Flux.java:8402) at reactor.core.publisher.FluxFlatMap$FlatMapMain.onNext(FluxFlatMap.java:426) at io.micronaut.reactive.reactor.instrument.ReactorSubscriber.onNext(ReactorSubscriber.java:57) at org.akhq.middlewares.HttpServerAccessLogFilter$HttpServerPublisher$1.onNext(HttpServerAccessLogFilter.java:138) at org.akhq.middlewares.HttpServerAccessLogFilter$HttpServerPublisher$1.onNext(HttpServerAccessLogFilter.java:129) at reactor.core.publisher.StrictSubscriber.onNext(StrictSubscriber.java:89) at io.micronaut.reactive.reactor.instrument.ReactorSubscriber.onNext(ReactorSubscriber.java:57) at reactor.core.publisher.FluxOnErrorResume$ResumeSubscriber.onNext(FluxOnErrorResume.java:79) at io.micronaut.reactive.reactor.instrument.ReactorSubscriber.onNext(ReactorSubscriber.java:57) at reactor.core.publisher.FluxFlatMap$FlatMapMain.drainLoop(FluxFlatMap.java:712) at reactor.core.publisher.FluxFlatMap$FlatMapMain.drain(FluxFlatMap.java:588) at reactor.core.publisher.FluxFlatMap$FlatMapInner.onSubscribe(FluxFlatMap.java:955) at io.micronaut.reactive.reactor.instrument.ReactorSubscriber.onSubscribe(ReactorSubscriber.java:50) at reactor.core.publisher.FluxJust.subscribe(FluxJust.java:68) at reactor.core.publisher.Flux.subscribe(Flux.java:8402) at reactor.core.publisher.FluxFlatMap$FlatMapMain.onNext(FluxFlatMap.java:426) at io.micronaut.reactive.reactor.instrument.ReactorSubscriber.onNext(ReactorSubscriber.java:57) at reactor.core.publisher.FluxFlatMap$FlatMapMain.drainLoop(FluxFlatMap.java:712) at reactor.core.publisher.FluxFlatMap$FlatMapMain.drain(FluxFlatMap.java:588) at reactor.core.publisher.FluxFlatMap$FlatMapInner.onNext(FluxFlatMap.java:971) at io.micronaut.reactive.reactor.instrument.ReactorSubscriber.onNext(ReactorSubscriber.java:57) at reactor.core.publisher.FluxPublishOn$PublishOnSubscriber.runBackfused(FluxPublishOn.java:484) at reactor.core.publisher.FluxPublishOn$PublishOnSubscriber.run(FluxPublishOn.java:521) at io.micronaut.reactive.reactor.instrument.ReactorInstrumentation.lambda$null$0(ReactorInstrumentation.java:62) at reactor.core.scheduler.WorkerTask.call(WorkerTask.java:84) at reactor.core.scheduler.WorkerTask.call(WorkerTask.java:37) at io.micrometer.core.instrument.composite.CompositeTimer.recordCallable(CompositeTimer.java:68) at io.micrometer.core.instrument.Timer.lambda$wrap$1(Timer.java:171) at io.micronaut.scheduling.instrument.InvocationInstrumenterWrappedCallable.call(InvocationInstrumenterWrappedCallable.java:53) at java.base/java.util.concurrent.FutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source)
I tried to increase memory and heap size, but it didn't help (up to 14GB). Any ideas how can I limit the infinite tailing of a topic?
We are testing JAVA_TOOL_OPTIONS set to -XX:+UseG1GC -XX:+AlwaysActAsServerClassMachine for AHKQ hosted on our AKS environment.
So far it's working good for few days straight.
We have used this trick for other pods using java apps as well which faced similar OOM issues. Seems that it might help in this case as well.
Root cause: If there're less than 2 CPUs (in our case limit is less than 1 CPU) then serial GC is being used as default one. It seems that it's not good enough and ends up with OOM issues when handling heavy memory usage.
We too have frequently this issue, G1HC seems to be a better option for akhq and Java 17 greatly improve it, with a 20% reduction in memory footprint. @tchiotludo would you merge a PR that upggrade to Java 17 ? I can work on it.
Also, the issue is on direct memory (out of heap space) so it's strange that using G1GC solves the issue. More memory should be allocated but not more heap to let space for direct memory buffers.
As the container image is JRE it may be hard to investiguate as we misses JDK tools in the image to look at what's going on, I also would like to propose to switch to JDK images instead of JRE one, again I can do this in a PR to migrate to Java 17.
By the way, @MichalPopielski -XX:+AlwaysActAsServerClassMachine should not be needed if the heap size is more than 1GB as the JVM will switch to server class machine automatically in this case.
@loicmathieu java 17 could make sense I think, go for it. For the jdk one, it will make the image really heavy, why not published on alternative image with jdk that we will ask people to use on debugging purpose ?
Dear All, New OOO error, occurred once again. To solve it we had to restart the whole pod, otherwise users were receiving grey background only. All Java Options were set as in my previous post.
stacktrace
2022-11-29 08:51:12,319 ERROR -thread-95 i.m.h.server.RouteExecutor Unexpected error occurred: Direct buffer memory java.lang.OutOfMemoryError: Direct buffer memory at java.base/java.nio.Bits.reserveMemory(Unknown Source) at java.base/java.nio.DirectByteBuffer.
(Unknown Source) at java.base/java.nio.ByteBuffer.allocateDirect(Unknown Source) at io.netty.buffer.PoolArena$DirectArena.allocateDirect(PoolArena.java:649) at io.netty.buffer.PoolArena$DirectArena.newChunk(PoolArena.java:624) at io.netty.buffer.PoolArena.allocateNormal(PoolArena.java:203) at io.netty.buffer.PoolArena.tcacheAllocateSmall(PoolArena.java:173) at io.netty.buffer.PoolArena.allocate(PoolArena.java:134) at io.netty.buffer.PoolArena.allocate(PoolArena.java:126) at io.netty.buffer.PooledByteBufAllocator.newDirectBuffer(PooledByteBufAllocator.java:396) at io.netty.buffer.AbstractByteBufAllocator.directBuffer(AbstractByteBufAllocator.java:188) at io.netty.buffer.AbstractByteBufAllocator.directBuffer(AbstractByteBufAllocator.java:174) at io.netty.buffer.AbstractByteBufAllocator.buffer(AbstractByteBufAllocator.java:108) at io.micronaut.buffer.netty.NettyByteBufferFactory.buffer(NettyByteBufferFactory.java:77) at io.micronaut.json.codec.MapperMediaTypeCodec.encode(MapperMediaTypeCodec.java:266) at io.micronaut.http.server.netty.RoutingInBoundHandler.encodeBodyAsByteBuf(RoutingInBoundHandler.java:1358) at io.micronaut.http.server.netty.RoutingInBoundHandler.encodeBodyWithCodec(RoutingInBoundHandler.java:1299) at io.micronaut.http.server.netty.RoutingInBoundHandler.encodeResponseBody(RoutingInBoundHandler.java:1117) at io.micronaut.http.server.netty.RoutingInBoundHandler.encodeHttpResponse(RoutingInBoundHandler.java:962) at io.micronaut.http.server.netty.RoutingInBoundHandler.access$000(RoutingInBoundHandler.java:144) at io.micronaut.http.server.netty.RoutingInBoundHandler$2.doOnNext(RoutingInBoundHandler.java:579) at io.micronaut.http.server.netty.RoutingInBoundHandler$2.doOnNext(RoutingInBoundHandler.java:571) at io.micronaut.core.async.subscriber.CompletionAwareSubscriber.onNext(CompletionAwareSubscriber.java:52) at reactor.core.publisher.StrictSubscriber.onNext(StrictSubscriber.java:89) at io.micronaut.reactive.reactor.instrument.ReactorSubscriber.onNext(ReactorSubscriber.java:57) at reactor.core.publisher.FluxContextWrite$ContextWriteSubscriber.onNext(FluxContextWrite.java:107) at io.micronaut.reactive.reactor.instrument.ReactorSubscriber.onNext(ReactorSubscriber.java:57) at reactor.core.publisher.FluxOnErrorResume$ResumeSubscriber.onNext(FluxOnErrorResume.java:79) at io.micronaut.reactive.reactor.instrument.ReactorSubscriber.onNext(ReactorSubscriber.java:57) at reactor.core.publisher.FluxFlatMap$FlatMapMain.drainLoop(FluxFlatMap.java:712) at reactor.core.publisher.FluxFlatMap$FlatMapMain.drain(FluxFlatMap.java:588) at reactor.core.publisher.FluxFlatMap$FlatMapInner.onSubscribe(FluxFlatMap.java:955) at io.micronaut.reactive.reactor.instrument.ReactorSubscriber.onSubscribe(ReactorSubscriber.java:50) at reactor.core.publisher.FluxJust.subscribe(FluxJust.java:68) at reactor.core.publisher.Flux.subscribe(Flux.java:8402) at reactor.core.publisher.FluxFlatMap$FlatMapMain.onNext(FluxFlatMap.java:426) at io.micronaut.reactive.reactor.instrument.ReactorSubscriber.onNext(ReactorSubscriber.java:57) at reactor.core.publisher.FluxOnErrorResume$ResumeSubscriber.onNext(FluxOnErrorResume.java:79) at io.micronaut.reactive.reactor.instrument.ReactorSubscriber.onNext(ReactorSubscriber.java:57) at reactor.core.publisher.FluxFlatMap$FlatMapMain.drainLoop(FluxFlatMap.java:712) at reactor.core.publisher.FluxFlatMap$FlatMapMain.drain(FluxFlatMap.java:588) at reactor.core.publisher.FluxFlatMap$FlatMapInner.onSubscribe(FluxFlatMap.java:955) at io.micronaut.reactive.reactor.instrument.ReactorSubscriber.onSubscribe(ReactorSubscriber.java:50) at reactor.core.publisher.FluxJust.subscribe(FluxJust.java:68) at reactor.core.publisher.Flux.subscribe(Flux.java:8402) at reactor.core.publisher.FluxFlatMap$FlatMapMain.onNext(FluxFlatMap.java:426) at io.micronaut.configuration.metrics.binder.web.WebMetricsPublisher$1.onNext(WebMetricsPublisher.java:180) at io.micronaut.configuration.metrics.binder.web.WebMetricsPublisher$1.onNext(WebMetricsPublisher.java:157) at io.micronaut.reactive.reactor.instrument.ReactorSubscriber.onNext(ReactorSubscriber.java:57) at reactor.core.publisher.FluxOnErrorResume$ResumeSubscriber.onNext(FluxOnErrorResume.java:79) at io.micronaut.reactive.reactor.instrument.ReactorSubscriber.onNext(ReactorSubscriber.java:57) at reactor.core.publisher.FluxFlatMap$FlatMapMain.drainLoop(FluxFlatMap.java:712) at reactor.core.publisher.FluxFlatMap$FlatMapMain.drain(FluxFlatMap.java:588) at reactor.core.publisher.FluxFlatMap$FlatMapInner.onSubscribe(FluxFlatMap.java:955) at io.micronaut.reactive.reactor.instrument.ReactorSubscriber.onSubscribe(ReactorSubscriber.java:50) at reactor.core.publisher.FluxJust.subscribe(FluxJust.java:68) at reactor.core.publisher.Flux.subscribe(Flux.java:8402) at reactor.core.publisher.FluxFlatMap$FlatMapMain.onNext(FluxFlatMap.java:426) at io.micronaut.reactive.reactor.instrument.ReactorSubscriber.onNext(ReactorSubscriber.java:57) at reactor.core.publisher.FluxSwitchIfEmpty$SwitchIfEmptySubscriber.onNext(FluxSwitchIfEmpty.java:74) at io.micronaut.reactive.reactor.instrument.ReactorSubscriber.onNext(ReactorSubscriber.java:57) at reactor.core.publisher.MonoNext$NextSubscriber.onNext(MonoNext.java:82) at io.micronaut.reactive.reactor.instrument.ReactorSubscriber.onNext(ReactorSubscriber.java:57) at reactor.core.publisher.FluxSwitchIfEmpty$SwitchIfEmptySubscriber.onNext(FluxSwitchIfEmpty.java:74) at io.micronaut.reactive.reactor.instrument.ReactorSubscriber.onNext(ReactorSubscriber.java:57) at reactor.core.publisher.MonoFlatMapMany$FlatMapManyInner.onNext(MonoFlatMapMany.java:250) at io.micronaut.reactive.reactor.instrument.ReactorSubscriber.onNext(ReactorSubscriber.java:57) at reactor.core.publisher.FluxOnErrorResume$ResumeSubscriber.onNext(FluxOnErrorResume.java:79) at io.micronaut.reactive.reactor.instrument.ReactorSubscriber.onNext(ReactorSubscriber.java:57) at reactor.core.publisher.FluxFlatMap$FlatMapMain.drainLoop(FluxFlatMap.java:712) at reactor.core.publisher.FluxFlatMap$FlatMapMain.drain(FluxFlatMap.java:588) at reactor.core.publisher.FluxFlatMap$FlatMapInner.onSubscribe(FluxFlatMap.java:955) at io.micronaut.reactive.reactor.instrument.ReactorSubscriber.onSubscribe(ReactorSubscriber.java:50) at reactor.core.publisher.FluxJust.subscribe(FluxJust.java:68) at reactor.core.publisher.Flux.subscribe(Flux.java:8402) at reactor.core.publisher.FluxFlatMap$FlatMapMain.onNext(FluxFlatMap.java:426) at io.micronaut.reactive.reactor.instrument.ReactorSubscriber.onNext(ReactorSubscriber.java:57) at org.akhq.middlewares.HttpServerAccessLogFilter$HttpServerPublisher$1.onNext(HttpServerAccessLogFilter.java:138) at org.akhq.middlewares.HttpServerAccessLogFilter$HttpServerPublisher$1.onNext(HttpServerAccessLogFilter.java:129) at reactor.core.publisher.StrictSubscriber.onNext(StrictSubscriber.java:89) at io.micronaut.reactive.reactor.instrument.ReactorSubscriber.onNext(ReactorSubscriber.java:57) at reactor.core.publisher.FluxOnErrorResume$ResumeSubscriber.onNext(FluxOnErrorResume.java:79) at io.micronaut.reactive.reactor.instrument.ReactorSubscriber.onNext(ReactorSubscriber.java:57) at reactor.core.publisher.FluxFlatMap$FlatMapMain.drainLoop(FluxFlatMap.java:712) at reactor.core.publisher.FluxFlatMap$FlatMapMain.drain(FluxFlatMap.java:588) at reactor.core.publisher.FluxFlatMap$FlatMapInner.onSubscribe(FluxFlatMap.java:955) at io.micronaut.reactive.reactor.instrument.ReactorSubscriber.onSubscribe(ReactorSubscriber.java:50) at reactor.core.publisher.FluxJust.subscribe(FluxJust.java:68) at reactor.core.publisher.Flux.subscribe(Flux.java:8402) at reactor.core.publisher.FluxFlatMap$FlatMapMain.onNext(FluxFlatMap.java:426) at io.micronaut.reactive.reactor.instrument.ReactorSubscriber.onNext(ReactorSubscriber.java:57) at reactor.core.publisher.FluxFlatMap$FlatMapMain.drainLoop(FluxFlatMap.java:712) at reactor.core.publisher.FluxFlatMap$FlatMapMain.drain(FluxFlatMap.java:588) at reactor.core.publisher.FluxFlatMap$FlatMapInner.onNext(FluxFlatMap.java:971) at io.micronaut.reactive.reactor.instrument.ReactorSubscriber.onNext(ReactorSubscriber.java:57) at reactor.core.publisher.FluxPublishOn$PublishOnSubscriber.runBackfused(FluxPublishOn.java:484) at reactor.core.publisher.FluxPublishOn$PublishOnSubscriber.run(FluxPublishOn.java:521) at io.micronaut.reactive.reactor.instrument.ReactorInstrumentation.lambda$null$0(ReactorInstrumentation.java:62) at reactor.core.scheduler.WorkerTask.call(WorkerTask.java:84) at reactor.core.scheduler.WorkerTask.call(WorkerTask.java:37) at io.micrometer.core.instrument.composite.CompositeTimer.recordCallable(CompositeTimer.java:68) at io.micrometer.core.instrument.Timer.lambda$wrap$1(Timer.java:171) at io.micronaut.scheduling.instrument.InvocationInstrumenterWrappedCallable.call(InvocationInstrumenterWrappedCallable.java:53) at java.base/java.util.concurrent.FutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) Exception in thread "io-executor-thread-95" java.lang.OutOfMemoryError: Direct buffer memory at java.base/java.nio.Bits.reserveMemory(Unknown Source) at java.base/java.nio.DirectByteBuffer. (Unknown Source) at java.base/java.nio.ByteBuffer.allocateDirect(Unknown Source) at io.netty.buffer.PoolArena$DirectArena.allocateDirect(PoolArena.java:649) at io.netty.buffer.PoolArena$DirectArena.newChunk(PoolArena.java:624) at io.netty.buffer.PoolArena.allocateNormal(PoolArena.java:203) at io.netty.buffer.PoolArena.tcacheAllocateSmall(PoolArena.java:173) at io.netty.buffer.PoolArena.allocate(PoolArena.java:134) at io.netty.buffer.PoolArena.allocate(PoolArena.java:126) at io.netty.buffer.PooledByteBufAllocator.newDirectBuffer(PooledByteBufAllocator.java:396) at io.netty.buffer.AbstractByteBufAllocator.directBuffer(AbstractByteBufAllocator.java:188) at io.netty.buffer.AbstractByteBufAllocator.directBuffer(AbstractByteBufAllocator.java:174) at io.netty.buffer.AbstractByteBufAllocator.buffer(AbstractByteBufAllocator.java:108) at io.micronaut.buffer.netty.NettyByteBufferFactory.buffer(NettyByteBufferFactory.java:77) at io.micronaut.json.codec.MapperMediaTypeCodec.encode(MapperMediaTypeCodec.java:255) at io.micronaut.http.server.netty.RoutingInBoundHandler.encodeBodyAsByteBuf(RoutingInBoundHandler.java:1360) at io.micronaut.http.server.netty.RoutingInBoundHandler.encodeBodyWithCodec(RoutingInBoundHandler.java:1299) at io.micronaut.http.server.netty.RoutingInBoundHandler.encodeResponseBody(RoutingInBoundHandler.java:1117) at io.micronaut.http.server.netty.RoutingInBoundHandler.encodeHttpResponse(RoutingInBoundHandler.java:962) at io.micronaut.http.server.netty.RoutingInBoundHandler.access$000(RoutingInBoundHandler.java:144) at io.micronaut.http.server.netty.RoutingInBoundHandler$2.doOnError(RoutingInBoundHandler.java:593) at io.micronaut.core.async.subscriber.CompletionAwareSubscriber.onError(CompletionAwareSubscriber.java:63) at io.micronaut.core.async.subscriber.CompletionAwareSubscriber.onNext(CompletionAwareSubscriber.java:54) at reactor.core.publisher.StrictSubscriber.onNext(StrictSubscriber.java:89) at io.micronaut.reactive.reactor.instrument.ReactorSubscriber.onNext(ReactorSubscriber.java:57) at reactor.core.publisher.FluxContextWrite$ContextWriteSubscriber.onNext(FluxContextWrite.java:107) at io.micronaut.reactive.reactor.instrument.ReactorSubscriber.onNext(ReactorSubscriber.java:57) at reactor.core.publisher.FluxOnErrorResume$ResumeSubscriber.onNext(FluxOnErrorResume.java:79) at io.micronaut.reactive.reactor.instrument.ReactorSubscriber.onNext(ReactorSubscriber.java:57) at reactor.core.publisher.FluxFlatMap$FlatMapMain.drainLoop(FluxFlatMap.java:712) at reactor.core.publisher.FluxFlatMap$FlatMapMain.drain(FluxFlatMap.java:588) at reactor.core.publisher.FluxFlatMap$FlatMapInner.onSubscribe(FluxFlatMap.java:955) at io.micronaut.reactive.reactor.instrument.ReactorSubscriber.onSubscribe(ReactorSubscriber.java:50) at reactor.core.publisher.FluxJust.subscribe(FluxJust.java:68) at reactor.core.publisher.Flux.subscribe(Flux.java:8402) at reactor.core.publisher.FluxFlatMap$FlatMapMain.onNext(FluxFlatMap.java:426) at io.micronaut.reactive.reactor.instrument.ReactorSubscriber.onNext(ReactorSubscriber.java:57) at reactor.core.publisher.FluxOnErrorResume$ResumeSubscriber.onNext(FluxOnErrorResume.java:79) at io.micronaut.reactive.reactor.instrument.ReactorSubscriber.onNext(ReactorSubscriber.java:57) at reactor.core.publisher.FluxFlatMap$FlatMapMain.drainLoop(FluxFlatMap.java:712) at reactor.core.publisher.FluxFlatMap$FlatMapMain.drain(FluxFlatMap.java:588) at reactor.core.publisher.FluxFlatMap$FlatMapInner.onSubscribe(FluxFlatMap.java:955) at io.micronaut.reactive.reactor.instrument.ReactorSubscriber.onSubscribe(ReactorSubscriber.java:50) at reactor.core.publisher.FluxJust.subscribe(FluxJust.java:68) at reactor.core.publisher.Flux.subscribe(Flux.java:8402) at reactor.core.publisher.FluxFlatMap$FlatMapMain.onNext(FluxFlatMap.java:426) at io.micronaut.configuration.metrics.binder.web.WebMetricsPublisher$1.onNext(WebMetricsPublisher.java:180) at io.micronaut.configuration.metrics.binder.web.WebMetricsPublisher$1.onNext(WebMetricsPublisher.java:157) at io.micronaut.reactive.reactor.instrument.ReactorSubscriber.onNext(ReactorSubscriber.java:57) at reactor.core.publisher.FluxOnErrorResume$ResumeSubscriber.onNext(FluxOnErrorResume.java:79) at io.micronaut.reactive.reactor.instrument.ReactorSubscriber.onNext(ReactorSubscriber.java:57) at reactor.core.publisher.FluxFlatMap$FlatMapMain.drainLoop(FluxFlatMap.java:712) at reactor.core.publisher.FluxFlatMap$FlatMapMain.drain(FluxFlatMap.java:588) at reactor.core.publisher.FluxFlatMap$FlatMapInner.onSubscribe(FluxFlatMap.java:955) at io.micronaut.reactive.reactor.instrument.ReactorSubscriber.onSubscribe(ReactorSubscriber.java:50) at reactor.core.publisher.FluxJust.subscribe(FluxJust.java:68) at reactor.core.publisher.Flux.subscribe(Flux.java:8402) at reactor.core.publisher.FluxFlatMap$FlatMapMain.onNext(FluxFlatMap.java:426) at io.micronaut.reactive.reactor.instrument.ReactorSubscriber.onNext(ReactorSubscriber.java:57) at reactor.core.publisher.FluxSwitchIfEmpty$SwitchIfEmptySubscriber.onNext(FluxSwitchIfEmpty.java:74) at io.micronaut.reactive.reactor.instrument.ReactorSubscriber.onNext(ReactorSubscriber.java:57) at reactor.core.publisher.MonoNext$NextSubscriber.onNext(MonoNext.java:82) at io.micronaut.reactive.reactor.instrument.ReactorSubscriber.onNext(ReactorSubscriber.java:57) at reactor.core.publisher.FluxSwitchIfEmpty$SwitchIfEmptySubscriber.onNext(FluxSwitchIfEmpty.java:74) at io.micronaut.reactive.reactor.instrument.ReactorSubscriber.onNext(ReactorSubscriber.java:57) at reactor.core.publisher.MonoFlatMapMany$FlatMapManyInner.onNext(MonoFlatMapMany.java:250) at io.micronaut.reactive.reactor.instrument.ReactorSubscriber.onNext(ReactorSubscriber.java:57) at reactor.core.publisher.FluxOnErrorResume$ResumeSubscriber.onNext(FluxOnErrorResume.java:79) at io.micronaut.reactive.reactor.instrument.ReactorSubscriber.onNext(ReactorSubscriber.java:57) at reactor.core.publisher.FluxFlatMap$FlatMapMain.drainLoop(FluxFlatMap.java:712) at reactor.core.publisher.FluxFlatMap$FlatMapMain.drain(FluxFlatMap.java:588) at reactor.core.publisher.FluxFlatMap$FlatMapInner.onSubscribe(FluxFlatMap.java:955) at io.micronaut.reactive.reactor.instrument.ReactorSubscriber.onSubscribe(ReactorSubscriber.java:50) at reactor.core.publisher.FluxJust.subscribe(FluxJust.java:68) at reactor.core.publisher.Flux.subscribe(Flux.java:8402) at reactor.core.publisher.FluxFlatMap$FlatMapMain.onNext(FluxFlatMap.java:426) at io.micronaut.reactive.reactor.instrument.ReactorSubscriber.onNext(ReactorSubscriber.java:57) at org.akhq.middlewares.HttpServerAccessLogFilter$HttpServerPublisher$1.onNext(HttpServerAccessLogFilter.java:138) at org.akhq.middlewares.HttpServerAccessLogFilter$HttpServerPublisher$1.onNext(HttpServerAccessLogFilter.java:129) at reactor.core.publisher.StrictSubscriber.onNext(StrictSubscriber.java:89) at io.micronaut.reactive.reactor.instrument.ReactorSubscriber.onNext(ReactorSubscriber.java:57) at reactor.core.publisher.FluxOnErrorResume$ResumeSubscriber.onNext(FluxOnErrorResume.java:79) at io.micronaut.reactive.reactor.instrument.ReactorSubscriber.onNext(ReactorSubscriber.java:57) at reactor.core.publisher.FluxFlatMap$FlatMapMain.drainLoop(FluxFlatMap.java:712) at reactor.core.publisher.FluxFlatMap$FlatMapMain.drain(FluxFlatMap.java:588) at reactor.core.publisher.FluxFlatMap$FlatMapInner.onSubscribe(FluxFlatMap.java:955) at io.micronaut.reactive.reactor.instrument.ReactorSubscriber.onSubscribe(ReactorSubscriber.java:50) at reactor.core.publisher.FluxJust.subscribe(FluxJust.java:68) at reactor.core.publisher.Flux.subscribe(Flux.java:8402) at reactor.core.publisher.FluxFlatMap$FlatMapMain.onNext(FluxFlatMap.java:426) at io.micronaut.reactive.reactor.instrument.ReactorSubscriber.onNext(ReactorSubscriber.java:57) at reactor.core.publisher.FluxFlatMap$FlatMapMain.drainLoop(FluxFlatMap.java:712) at reactor.core.publisher.FluxFlatMap$FlatMapMain.drain(FluxFlatMap.java:588) at reactor.core.publisher.FluxFlatMap$FlatMapInner.onNext(FluxFlatMap.java:971) at io.micronaut.reactive.reactor.instrument.ReactorSubscriber.onNext(ReactorSubscriber.java:57) at reactor.core.publisher.FluxPublishOn$PublishOnSubscriber.runBackfused(FluxPublishOn.java:484) at reactor.core.publisher.FluxPublishOn$PublishOnSubscriber.run(FluxPublishOn.java:521) at io.micronaut.reactive.reactor.instrument.ReactorInstrumentation.lambda$null$0(ReactorInstrumentation.java:62) at reactor.core.scheduler.WorkerTask.call(WorkerTask.java:84) at reactor.core.scheduler.WorkerTask.call(WorkerTask.java:37) at io.micrometer.core.instrument.composite.CompositeTimer.recordCallable(CompositeTimer.java:68) at io.micrometer.core.instrument.Timer.lambda$wrap$1(Timer.java:171) at io.micronaut.scheduling.instrument.InvocationInstrumenterWrappedCallable.call(InvocationInstrumenterWrappedCallable.java:53) at java.base/java.util.concurrent.FutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source)
Health metric monitoring such case could help to automate the restart of AKHQ which ran out of memory
We are also seeing OOM exceptions which kill the background processes but does not cause a health check to fail. This causes our monitoring to think akhq is still available and blocks auto healing of the pod in kubernetes. It would be great if the app crashes or at least has a failing health check when only the frontend can be served.
