quarkus-operator-sdk
quarkus-operator-sdk copied to clipboard
Published
20 hours ago •
quarkiverse
quarkiverse
Dev & Continuous Testing modes fails with too many open files
trafficstars
After a few (10-15) test re-runs and/or restarts the application fails with "too many open files".
The error is when the Fabric8 K8s client is attempting to initialize; it seems like it might be leaking K8s clients upon restart.
Do you have a stack trace? What exactly causes the issue? Running in dev mode? Something else?
2023-12-12 20:50:24,985 ERROR [io.qua.dep.dev.IsolatedDevModeMain] (vert.x-worker-thread-2) Failed to start quarkus: io.quarkus.dev.appstate.ApplicationStartException: java.lang.RuntimeException: Failed to start quarkus
at io.quarkus.dev.appstate.ApplicationStateNotification.waitForApplicationStart(ApplicationStateNotification.java:58)
at io.quarkus.runner.bootstrap.StartupActionImpl.runMainClass(StartupActionImpl.java:132)
at io.quarkus.deployment.dev.IsolatedDevModeMain.restartApp(IsolatedDevModeMain.java:193)
at io.quarkus.deployment.dev.IsolatedDevModeMain.restartCallback(IsolatedDevModeMain.java:174)
at io.quarkus.deployment.dev.RuntimeUpdatesProcessor.doScan(RuntimeUpdatesProcessor.java:541)
at io.quarkus.deployment.dev.RuntimeUpdatesProcessor.doScan(RuntimeUpdatesProcessor.java:441)
at io.quarkus.vertx.http.runtime.devmode.VertxHttpHotReplacementSetup$5.call(VertxHttpHotReplacementSetup.java:150)
at io.quarkus.vertx.http.runtime.devmode.VertxHttpHotReplacementSetup$5.call(VertxHttpHotReplacementSetup.java:137)
at io.vertx.core.impl.ContextBase.lambda$executeBlocking$0(ContextBase.java:167)
at io.vertx.core.impl.ContextInternal.dispatch(ContextInternal.java:277)
at io.vertx.core.impl.ContextBase.lambda$internalExecuteBlocking$2(ContextBase.java:199)
at org.jboss.threads.ContextHandler$1.runWith(ContextHandler.java:18)
at org.jboss.threads.EnhancedQueueExecutor$Task.run(EnhancedQueueExecutor.java:2513)
at org.jboss.threads.EnhancedQueueExecutor$ThreadBody.run(EnhancedQueueExecutor.java:1538)
at org.jboss.threads.DelegatingRunnable.run(DelegatingRunnable.java:29)
at org.jboss.threads.ThreadLocalResettingRunnable.run(ThreadLocalResettingRunnable.java:29)
at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
at java.base/java.lang.Thread.run(Thread.java:833)
Suppressed: java.lang.reflect.InvocationTargetException
at jdk.internal.reflect.GeneratedMethodAccessor1725.invoke(Unknown Source)
at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:568)
at io.quarkus.runner.bootstrap.StartupActionImpl.runMainClass(StartupActionImpl.java:161)
... 16 more
Caused by: java.lang.NullPointerException: Cannot invoke "io.quarkus.arc.ArcContainer.instance(java.lang.Class, java.lang.annotation.Annotation[])" because "container" is null
at io.quarkus.vault.runtime.config.VaultConfigSourceFactory.getConfigSources(VaultConfigSourceFactory.java:19)
at io.quarkus.vault.runtime.config.VaultConfigSourceFactory.getConfigSources(VaultConfigSourceFactory.java:14)
at io.smallrye.config.ConfigSourceFactory$ConfigurableConfigSourceFactory.getConfigSources(ConfigSourceFactory.java:58)
at io.smallrye.config.ConfigurableConfigSource.getConfigSources(ConfigurableConfigSource.java:50)
at io.smallrye.config.SmallRyeConfig$ConfigSources.mapLateSources(SmallRyeConfig.java:687)
at io.smallrye.config.SmallRyeConfig$ConfigSources.<init>(SmallRyeConfig.java:577)
at io.smallrye.config.SmallRyeConfig.<init>(SmallRyeConfig.java:68)
at io.smallrye.config.SmallRyeConfigBuilder.build(SmallRyeConfigBuilder.java:698)
at io.quarkus.runtime.generated.Config.readConfig(Unknown Source)
at io.quarkus.runtime.generated.Config.createRunTimeConfig(Unknown Source)
... 20 more
Caused by: java.lang.RuntimeException: Failed to start quarkus
at io.quarkus.runner.ApplicationImpl.doStart(Unknown Source)
at io.quarkus.runtime.Application.start(Application.java:101)
at io.quarkus.runtime.ApplicationLifecycleManager.run(ApplicationLifecycleManager.java:111)
at io.quarkus.runtime.Quarkus.run(Quarkus.java:71)
at io.quarkus.runtime.Quarkus.run(Quarkus.java:44)
at io.quarkus.runtime.Quarkus.run(Quarkus.java:124)
at io.quarkus.runner.GeneratedMain.main(Unknown Source)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:568)
at io.quarkus.runner.bootstrap.StartupActionImpl$1.run(StartupActionImpl.java:113)
... 1 more
Caused by: java.util.ServiceConfigurationError: io.fabric8.kubernetes.client.http.HttpClient$Factory: Provider io.fabric8.kubernetes.client.vertx.VertxHttpClientFactory could not be instantiated
at java.base/java.util.ServiceLoader.fail(ServiceLoader.java:586)
at java.base/java.util.ServiceLoader$ProviderImpl.newInstance(ServiceLoader.java:813)
at java.base/java.util.ServiceLoader$ProviderImpl.get(ServiceLoader.java:729)
at java.base/java.util.ServiceLoader$3.next(ServiceLoader.java:1403)
at java.base/java.lang.Iterable.forEach(Iterable.java:74)
at io.fabric8.kubernetes.client.utils.HttpClientUtils.getFactory(HttpClientUtils.java:151)
at io.fabric8.kubernetes.client.utils.HttpClientUtils.getHttpClientFactory(HttpClientUtils.java:136)
at io.fabric8.kubernetes.client.KubernetesClientBuilder.build(KubernetesClientBuilder.java:77)
at io.quarkus.it.openshift.client.runtime.OpenShiftClientProducer.openShiftClient(OpenShiftClientProducer.java:35)
at io.quarkus.it.openshift.client.runtime.OpenShiftClientProducer_ProducerMethod_openShiftClient_b751648494c051f2e6c0aaba5b3ccc0898717f6e_Bean.doCreate(Unknown Source)
at io.quarkus.it.openshift.client.runtime.OpenShiftClientProducer_ProducerMethod_openShiftClient_b751648494c051f2e6c0aaba5b3ccc0898717f6e_Bean.create(Unknown Source)
at io.quarkus.it.openshift.client.runtime.OpenShiftClientProducer_ProducerMethod_openShiftClient_b751648494c051f2e6c0aaba5b3ccc0898717f6e_Bean.create(Unknown Source)
at io.quarkus.arc.impl.AbstractSharedContext.createInstanceHandle(AbstractSharedContext.java:113)
at io.quarkus.arc.impl.AbstractSharedContext$1.get(AbstractSharedContext.java:37)
at io.quarkus.arc.impl.AbstractSharedContext$1.get(AbstractSharedContext.java:34)
at io.quarkus.arc.impl.LazyValue.get(LazyValue.java:32)
at io.quarkus.arc.impl.ComputingCache.computeIfAbsent(ComputingCache.java:69)
at io.quarkus.arc.impl.AbstractSharedContext.get(AbstractSharedContext.java:34)
at io.quarkus.it.openshift.client.runtime.OpenShiftClientProducer_ProducerMethod_openShiftClient_b751648494c051f2e6c0aaba5b3ccc0898717f6e_Bean.get(Unknown Source)
at io.quarkus.it.openshift.client.runtime.OpenShiftClientProducer_ProducerMethod_openShiftClient_b751648494c051f2e6c0aaba5b3ccc0898717f6e_Bean.get(Unknown Source)
at io.quarkus.arc.impl.ArcContainerImpl.beanInstanceHandle(ArcContainerImpl.java:539)
at io.quarkus.arc.impl.ArcContainerImpl.beanInstanceHandle(ArcContainerImpl.java:519)
at io.quarkus.arc.impl.ArcContainerImpl.beanInstanceHandle(ArcContainerImpl.java:552)
at io.quarkus.arc.impl.ArcContainerImpl.instanceHandle(ArcContainerImpl.java:514)
at io.quarkus.arc.impl.ArcContainerImpl.instance(ArcContainerImpl.java:279)
at io.quarkiverse.operatorsdk.runtime.ConfigurationServiceRecorder.lambda$configurationServiceSupplier$1(ConfigurationServiceRecorder.java:97)
at io.quarkus.arc.runtime.ArcRecorder$4.apply(ArcRecorder.java:129)
at io.quarkus.arc.runtime.ArcRecorder$4.apply(ArcRecorder.java:126)
at io.quarkiverse.operatorsdk.runtime.QuarkusConfigurationService_81b6261ab616dc08f5b67e5e42b894b7028ca2cb_Synthetic_Bean.createSynthetic(Unknown Source)
at io.quarkiverse.operatorsdk.runtime.QuarkusConfigurationService_81b6261ab616dc08f5b67e5e42b894b7028ca2cb_Synthetic_Bean.doCreate(Unknown Source)
at io.quarkiverse.operatorsdk.runtime.QuarkusConfigurationService_81b6261ab616dc08f5b67e5e42b894b7028ca2cb_Synthetic_Bean.create(Unknown Source)
at io.quarkiverse.operatorsdk.runtime.QuarkusConfigurationService_81b6261ab616dc08f5b67e5e42b894b7028ca2cb_Synthetic_Bean.create(Unknown Source)
at io.quarkus.arc.impl.AbstractSharedContext.createInstanceHandle(AbstractSharedContext.java:113)
at io.quarkus.arc.impl.AbstractShared$1.get(AbstractSharedContext.java:37)
at io.quarkus.arc.impl.AbstractSharedContext$1.get(AbstractSharedContext.java:34)
at io.quarkus.arc.impl.LazyValue.get(LazyValue.java:32)
at io.quarkus.arc.impl.ComputingCache.computeIfAbsent(ComputingCache.java:69)
at io.quarkus.arc.impl.AbstractSharedContext.get(AbstractSharedContext.java:34)
at io.quarkiverse.operatorsdk.runtime.QuarkusConfigurationService_81b6261ab616dc08f5b67e5e42b894b7028ca2cb_Synthetic_Bean.get(Unknown Source)
at io.quarkiverse.operatorsdk.runtime.QuarkusConfigurationService_81b6261ab616dc08f5b67e5e42b894b7028ca2cb_Synthetic_Bean.get(Unknown Source)
at io.quarkiverse.operatorsdk.runtime.AppEventListener_Bean.doCreate(Unknown Source)
at io.quarkiverse.operatorsdk.runtime.AppEventListener_Bean.create(Unknown Source)
at io.quarkiverse.operatorsdk.runtime.AppEventListener_Bean.create(Unknown Source)
at io.quarkus.arc.impl.AbstractSharedContext.createInstanceHandle(AbstractSharedContext.java:113)
at io.quarkus.arc.impl.AbstractSharedContext$1.get(AbstractSharedContext.java:37)
at io.quarkus.arc.impl.AbstractSharedContext$1.get(AbstractSharedContext.java:34)
at io.quarkus.arc.impl.LazyValue.get(LazyValue.java:32)
at io.quarkus.arc.impl.ComputingCache.computeIfAbsent(ComputingCache.java:69)
at io.quarkus.arc.impl.AbstractSharedContext.get(AbstractSharedContext.java:34)
at io.quarkiverse.operatorsdk.runtime.AppEventListener_Observer_onStartup_b28deb793825eb1808af096a843376083fea4592.notify(Unknown Source)
at io.quarkus.arc.impl.EventImpl$Notifier.notifyObservers(EventImpl.java:346)
at io.quarkus.arc.impl.EventImpl$Notifier.notify(EventImpl.java:328)
at io.quarkus.arc.impl.EventImpl.fire(EventImpl.java:82)
at io.quarkus.arc.runtime.ArcRecorder.fireLifecycleEvent(ArcRecorder.java:155)
at io.quarkus.arc.runtime.ArcRecorder.handleLifecycleEvents(ArcRecorder.java:106)
at io.quarkus.deployment.steps.LifecycleEventsBuildStep$startupEvent1144526294.deploy_0(Unknown Source)
at io.quarkus.deployment.steps.LifecycleEventsBuildStep$startupEvent1144526294.deploy(Unknown Source)
... 13 more
Caused by: java.lang.IllegalStateException: failed to create a child event loop
at io.netty.util.concurrent.MultithreadEventExecutorGroup.<init>(MultithreadEventExecutorGroup.java:88)
at io.netty.util.concurrent.MultithreadEventExecutorGroup.<init>(MultithreadEventExecutorGroup.java:60)
at io.netty.util.concurrent.MultithreadEventExecutorGroup.<init>(MultithreadEventExecutorGroup.java:49)
at io.netty.channel.MultithreadEventLoopGroup.<init>(MultithreadEventLoopGroup.java:59)
at io.netty.channel.nio.NioEventLoopGroup.<init>(NioEventLoopGroup.java:87)
at io.netty.channel.nio.NioEventLoopGroup.<init>(NioEventLoopGroup.java:82)
at io.netty.channel.nio.NioEventLoopGroup.<init>(NioEventLoopGroup.java:69)
at io.vertx.core.impl.transports.JDKTransport.eventLoopGroup(JDKTransport.java:34)
at io.vertx.core.impl.VertxImpl.<init>(VertxImpl.java:181)
at io.vertx.core.impl.VertxBuilder.vertx(VertxBuilder.java:229)
at io.vertx.core.Vertx.vertx(Vertx.java:88)
at io.fabric8.kubernetes.client.vertx.VertxHttpClientFactory.createVertxInstance(VertxHttpClientFactory.java:49)
at io.fabric8.kubernetes.client.vertx.VertxHttpClientFactory.<init>(VertxHttpClientFactory.java:32)
at jdk.internal.reflect.GeneratedConstructorAccessor297.newInstance(Unknown Source)
at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499)
at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480)
at java.base/java.util.ServiceLoader$ProviderImpl.newInstance(ServiceLoader.java:789)
... 68 more
Caused by: io.netty.channel.ChannelException: failed to open a new selector
at io.netty.channel.nio.NioEventLoop.openSelector(NioEventLoop.java:179)
at io.netty.channel.nio.NioEventLoop.<init>(NioEventLoop.java:146)
at io.netty.channel.nio.NioEventLoopGroup.newChild(NioEventLoopGroup.java:183)
at io.netty.channel.nio.NioEventLoopGroup.newChild(NioEventLoopGroup.java:38)
at io.netty.util.concurrent.MultithreadEventExecutorGroup.<init>(MultithreadEventExecutorGroup.java:84)
... 85 more
Caused by: java.io.IOException: Too many open files
at java.base/sun.nio.ch.KQueue.create(Native Method)
at java.base/sun.nio.ch.KQueueSelectorImpl.<init>(KQueueSelectorImpl.java:82)
at java.base/sun.nio.ch.KQueueSelectorProvider.openSelector(KQueueSelectorProvider.java:36)
at io.netty.channel.nio.NioEventLoop.openSelector(NioEventLoop.java:177)
... 89 more
Yes. This was while running dev mode only and making changes with it restarting while making changes.