vertx-mongo-client
vertx-mongo-client copied to clipboard
Published
20 hours ago •
vert-x3
vert-x3
Mongo client blocking event loop in vertx
We are using vertx mongo client in our application we are seeing lot of WARNINGS in our log with the following trace.
WARNING: Thread Thread[vert.x-eventloop-thread-1,5,main] has been blocked for 7963 ms, time limit is 2000
io.vertx.core.VertxException: Thread blocked
at java.lang.Iterable.forEach(Iterable.java:74)
at io.vertx.ext.mongo.impl.codec.json.JsonObjectCodec.forEach(JsonObjectCodec.java:119)
at io.vertx.ext.mongo.impl.codec.json.JsonObjectCodec.forEach(JsonObjectCodec.java:23)
at io.vertx.ext.mongo.impl.codec.json.AbstractJsonCodec.writeDocument(AbstractJsonCodec.java:258)
at io.vertx.ext.mongo.impl.codec.json.AbstractJsonCodec.writeValue(AbstractJsonCodec.java:105)
at io.vertx.ext.mongo.impl.codec.json.AbstractJsonCodec.lambda$writeArray$2(AbstractJsonCodec.java:298)
at io.vertx.ext.mongo.impl.codec.json.AbstractJsonCodec$$Lambda$234/1086686834.accept(Unknown Source)
at java.lang.Iterable.forEach(Iterable.java:75)
at io.vertx.ext.mongo.impl.codec.json.JsonObjectCodec.forEach(JsonObjectCodec.java:145)
at io.vertx.ext.mongo.impl.codec.json.JsonObjectCodec.forEach(JsonObjectCodec.java:23)
at io.vertx.ext.mongo.impl.codec.json.AbstractJsonCodec.writeArray(AbstractJsonCodec.java:298)
at io.vertx.ext.mongo.impl.codec.json.AbstractJsonCodec.writeValue(AbstractJsonCodec.java:90)
at io.vertx.ext.mongo.impl.codec.json.AbstractJsonCodec.lambda$writeDocument$1(AbstractJsonCodec.java:261)
at io.vertx.ext.mongo.impl.codec.json.AbstractJsonCodec$$Lambda$209/1377939936.accept(Unknown Source)
at io.vertx.ext.mongo.impl.codec.json.JsonObjectCodec.lambda$forEach$0(JsonObjectCodec.java:120)
at io.vertx.ext.mongo.impl.codec.json.JsonObjectCodec$$Lambda$210/126919379.accept(Unknown Source)
at java.lang.Iterable.forEach(Iterable.java:75)
at io.vertx.ext.mongo.impl.codec.json.JsonObjectCodec.forEach(JsonObjectCodec.java:119)
at io.vertx.ext.mongo.impl.codec.json.JsonObjectCodec.forEach(JsonObjectCodec.java:23)
at io.vertx.ext.mongo.impl.codec.json.AbstractJsonCodec.writeDocument(AbstractJsonCodec.java:258)
at io.vertx.ext.mongo.impl.codec.json.AbstractJsonCodec.writeValue(AbstractJsonCodec.java:105)
at io.vertx.ext.mongo.impl.codec.json.AbstractJsonCodec.lambda$writeArray$2(AbstractJsonCodec.java:298)
at io.vertx.ext.mongo.impl.codec.json.AbstractJsonCodec$$Lambda$234/1086686834.accept(Unknown Source)
at java.lang.Iterable.forEach(Iterable.java:75)
at io.vertx.ext.mongo.impl.codec.json.JsonObjectCodec.forEach(JsonObjectCodec.java:145)
at io.vertx.ext.mongo.impl.codec.json.JsonObjectCodec.forEach(JsonObjectCodec.java:23)
at io.vertx.ext.mongo.impl.codec.json.AbstractJsonCodec.writeArray(AbstractJsonCodec.java:298)
at io.vertx.ext.mongo.impl.codec.json.AbstractJsonCodec.writeValue(AbstractJsonCodec.java:90)
at io.vertx.ext.mongo.impl.codec.json.AbstractJsonCodec.lambda$writeDocument$1(AbstractJsonCodec.java:261)
at io.vertx.ext.mongo.impl.codec.json.AbstractJsonCodec$$Lambda$209/1377939936.accept(Unknown Source)
at io.vertx.ext.mongo.impl.codec.json.JsonObjectCodec.lambda$forEach$0(JsonObjectCodec.java:120)
at io.vertx.ext.mongo.impl.codec.json.JsonObjectCodec$$Lambda$210/126919379.accept(Unknown Source)
at java.lang.Iterable.forEach(Iterable.java:75)
at io.vertx.ext.mongo.impl.codec.json.JsonObjectCodec.forEach(JsonObjectCodec.java:119)
at io.vertx.ext.mongo.impl.codec.json.JsonObjectCodec.forEach(JsonObjectCodec.java:23)
at io.vertx.ext.mongo.impl.codec.json.AbstractJsonCodec.writeDocument(AbstractJsonCodec.java:258)
at io.vertx.ext.mongo.impl.codec.json.AbstractJsonCodec.encode(AbstractJsonCodec.java:28)
at org.bson.codecs.BsonDocumentWrapperCodec.encode(BsonDocumentWrapperCodec.java:63)
at org.bson.codecs.BsonDocumentWrapperCodec.encode(BsonDocumentWrapperCodec.java:29)
at com.mongodb.operation.BulkWriteBatch$WriteRequestEncoder.encode(BulkWriteBatch.java:381)
at com.mongodb.operation.BulkWriteBatch$WriteRequestEncoder.encode(BulkWriteBatch.java:371)
at org.bson.codecs.BsonDocumentWrapperCodec.encode(BsonDocumentWrapperCodec.java:63)
at org.bson.codecs.BsonDocumentWrapperCodec.encode(BsonDocumentWrapperCodec.java:29)
at com.mongodb.connection.BsonWriterHelper.writeDocument(BsonWriterHelper.java:74)
at com.mongodb.connection.BsonWriterHelper.writePayload(BsonWriterHelper.java:58)
at com.mongodb.connection.BsonWriterHelper.writePayloadArray(BsonWriterHelper.java:50)
at com.mongodb.connection.SplittablePayloadBsonWriter.writeEndDocument(SplittablePayloadBsonWriter.java:53)
at org.bson.codecs.BsonDocumentCodec.encode(BsonDocumentCodec.java:118)
at org.bson.codecs.BsonDocumentCodec.encode(BsonDocumentCodec.java:41)
at com.mongodb.connection.CommandMessage.addDocumentWithPayload(CommandMessage.java:166)
at com.mongodb.connection.CommandMessage.encodeMessageBodyWithMetadata(CommandMessage.java:148)
at com.mongodb.connection.RequestMessage.encode(RequestMessage.java:145)
at com.mongodb.connection.InternalStreamConnection.sendAndReceiveAsync(InternalStreamConnection.java:321)
at com.mongodb.connection.UsageTrackingInternalConnection.sendAndReceiveAsync(UsageTrackingInternalConnection.java:113)
at com.mongodb.connection.DefaultConnectionPool$PooledConnection.sendAndReceiveAsync(DefaultConnectionPool.java:452)
at com.mongodb.connection.CommandProtocolImpl.executeAsync(CommandProtocolImpl.java:91)
at com.mongodb.connection.DefaultServer$DefaultServerProtocolExecutor.executeAsync(DefaultServer.java:200)
at com.mongodb.connection.DefaultServerConnection.executeProtocolAsync(DefaultServerConnection.java:279)
at com.mongodb.connection.DefaultServerConnection.commandAsync(DefaultServerConnection.java:151)
at com.mongodb.operation.MixedBulkWriteOperation.executeCommandAsync(MixedBulkWriteOperation.java:382)
at com.mongodb.operation.MixedBulkWriteOperation.executeBatchesAsync(MixedBulkWriteOperation.java:314)
at com.mongodb.operation.MixedBulkWriteOperation.access$900(MixedBulkWriteOperation.java:66)
at com.mongodb.operation.MixedBulkWriteOperation$2$1.call(MixedBulkWriteOperation.java:233)
at com.mongodb.operation.OperationHelper.validateWriteRequests(OperationHelper.java:175)
at com.mongodb.operation.MixedBulkWriteOperation$2.call(MixedBulkWriteOperation.java:216)
at com.mongodb.operation.OperationHelper$7.onResult(OperationHelper.java:525)
at com.mongodb.operation.OperationHelper$7.onResult(OperationHelper.java:522)
at com.mongodb.connection.DefaultServer$1.onResult(DefaultServer.java:99)
at com.mongodb.connection.DefaultServer$1.onResult(DefaultServer.java:90)
at com.mongodb.internal.async.ErrorHandlingResultCallback.onResult(ErrorHandlingResultCallback.java:49)
at com.mongodb.connection.DefaultConnectionPool.openAsync(DefaultConnectionPool.java:189)
at com.mongodb.connection.DefaultConnectionPool.getAsync(DefaultConnectionPool.java:146)
at com.mongodb.connection.DefaultServer.getConnectionAsync(DefaultServer.java:90)
at com.mongodb.binding.AsyncClusterBinding$AsyncClusterBindingConnectionSource.getConnection(AsyncClusterBinding.java:114)
at com.mongodb.operation.OperationHelper.withConnectionSource(OperationHelper.java:522)
at com.mongodb.operation.OperationHelper.access$100(OperationHelper.java:61)
at com.mongodb.operation.OperationHelper$AsyncCallableWithConnectionAndSourceCallback.onResult(OperationHelper.java:542)
at com.mongodb.operation.OperationHelper$AsyncCallableWithConnectionAndSourceCallback.onResult(OperationHelper.java:530)
at com.mongodb.internal.async.ErrorHandlingResultCallback.onResult(ErrorHandlingResultCallback.java:49)
at com.mongodb.binding.AsyncClusterBinding$1.onResult(AsyncClusterBinding.java:88)
at com.mongodb.binding.AsyncClusterBinding$1.onResult(AsyncClusterBinding.java:82)
at com.mongodb.connection.BaseCluster$ServerSelectionRequest.onResult(BaseCluster.java:421)
at com.mongodb.connection.BaseCluster.handleServerSelectionRequest(BaseCluster.java:285)
at com.mongodb.connection.BaseCluster.selectServerAsync(BaseCluster.java:145)
at com.mongodb.binding.AsyncClusterBinding.getAsyncClusterBindingConnectionSource(AsyncClusterBinding.java:82)
at com.mongodb.binding.AsyncClusterBinding.getWriteConnectionSource(AsyncClusterBinding.java:77)
at com.mongodb.operation.OperationHelper.withConnection(OperationHelper.java:481)
at com.mongodb.operation.MixedBulkWriteOperation.executeAsync(MixedBulkWriteOperation.java:210)
at com.mongodb.async.client.AsyncOperationExecutorImpl$2.onResult(AsyncOperationExecutorImpl.java:100)
at com.mongodb.async.client.AsyncOperationExecutorImpl$2.onResult(AsyncOperationExecutorImpl.java:92)
at com.mongodb.async.client.ClientSessionHelper$2.onResult(ClientSessionHelper.java:73)
at com.mongodb.async.client.ClientSessionHelper$2.onResult(ClientSessionHelper.java:67)
at com.mongodb.connection.BaseCluster$ServerSelectionRequest.onResult(BaseCluster.java:421)
at com.mongodb.connection.BaseCluster.handleServerSelectionRequest(BaseCluster.java:285)
at com.mongodb.connection.BaseCluster.selectServerAsync(BaseCluster.java:145)
at com.mongodb.async.client.ClientSessionHelper.createClientSession(ClientSessionHelper.java:62)
at com.mongodb.async.client.ClientSessionHelper.withClientSession(ClientSessionHelper.java:48)
at com.mongodb.async.client.AsyncOperationExecutorImpl.execute(AsyncOperationExecutorImpl.java:92)
at com.mongodb.async.client.MongoCollectionImpl.executeSingleWriteRequest(MongoCollectionImpl.java:1114)
at com.mongodb.async.client.MongoCollectionImpl.executeInsertOne(MongoCollectionImpl.java:522)
at com.mongodb.async.client.MongoCollectionImpl.insertOne(MongoCollectionImpl.java:501)
at com.mongodb.async.client.MongoCollectionImpl.insertOne(MongoCollectionImpl.java:496)
at io.vertx.ext.mongo.impl.MongoClientImpl.saveWithOptions(MongoClientImpl.java:137)
at io.vertx.ext.mongo.impl.MongoClientImpl.save(MongoClientImpl.java:124)
at A.B.myPackage.sdk.dao.impl.mongo.L0MonitoringDataDAOImpl.create(L0MonitoringDataDAOImpl.java:32)
at A.B.myPackage.sdk.dao.facade.FacadeL0MonitoringDataDao$1.execute(FacadeL0MonitoringDataDao.java:31)
at A.B.myPackage.hbase.dao.facade.BaseFacadeDao.performSaveOperation(BaseFacadeDao.java:38)
at A.B.myPackage.sdk.dao.facade.FacadeL0MonitoringDataDao.create(FacadeL0MonitoringDataDao.java:28)
at A.B.myPackage.sdk.handler.L0MonitoringDataHandler.lambda$post$2(L0MonitoringDataHandler.java:101)
at A.B.myPackage.sdk.handler.L0MonitoringDataHandler$$Lambda$236/507063857.handle(Unknown Source)
at io.vertx.core.impl.FutureImpl.tryComplete(FutureImpl.java:125)
at io.vertx.core.impl.FutureImpl.complete(FutureImpl.java:86)
at A.B.myPackage.sdk.dao.impl.mongo.SdkUserDAOImpl.lambda$getSdkUserAsJson$1(SdkUserDAOImpl.java:51)
at A.B.myPackage.sdk.dao.impl.mongo.SdkUserDAOImpl$$Lambda$230/184006731.handle(Unknown Source)
at io.vertx.ext.mongo.impl.MongoClientImpl.lambda$null$11(MongoClientImpl.java:758)
at io.vertx.ext.mongo.impl.MongoClientImpl$$Lambda$211/367903824.handle(Unknown Source)
at io.vertx.core.impl.ContextImpl.lambda$wrapTask$2(ContextImpl.java:339)
at io.vertx.core.impl.ContextImpl$$Lambda$28/1005091648.run(Unknown Source)
at io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:163)
at io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:404)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:463)
at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:886)
at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
at java.lang.Thread.run(Thread.java:745)
Is mongo client blocking main event loop, because we have zookeeper configured which expects response from main event loop for every 30 seconds, The above error keeps on coming in our logs for every two seconds and application is getting out of zookeeper cluster. As event loop blocked zookeeper is removing my application node from the cluster.
Application Flow -> denotes success handler callback
Getcall from Collection A-> Insert record to Collection B -> Insert record to Collectoin C-> end routing context
