jsonde icon indicating copy to clipboard operation
jsonde copied to clipboard
verified publisher icon bedrin

Deadlock when instrumenting all code in Apache Tomcat 7.0.62

Open bedrin opened this issue 10 years ago • 0 comments

Full thread dump Java HotSpot(TM) 64-Bit Server VM (25.25-b02 mixed mode):

"jSonde-daemon-thread-4" #27 daemon prio=5 os_prio=0 tid=0x00000000575bb000 nid=0x1794 waiting on condition [0x000000005c18f000]
   java.lang.Thread.State: WAITING (parking)
        at sun.misc.Unsafe.park(Native Method)
        - parking to wait for  <0x00000000c3ea7de8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)

"jSonde-daemon-thread-3" #26 daemon prio=5 os_prio=0 tid=0x0000000057b74800 nid=0x1394 in Object.wait() [0x000000005bfaf000]
   java.lang.Thread.State: WAITING (on object monitor)
        at java.lang.Object.wait(Native Method)
        at java.lang.Object.wait(Object.java:502)
        at com.jsonde.profiler.network.NetworkServerImpl.sendMessage(NetworkServerImpl.java:160)
        - locked <0x00000000c3ea0ae0> (a com.jsonde.profiler.network.NetworkServerImpl)
        at com.jsonde.profiler.ProfilerImpl.sendMessage(ProfilerImpl.java:211)
        at com.jsonde.profiler.telemetry.TelemetryDataProvider.run(TelemetryDataProvider.java:42)
        at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
        at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:308)
        at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:180)
        at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:294)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)

"jSonde-daemon-thread-0" #21 daemon prio=5 os_prio=0 tid=0x0000000057b72800 nid=0x3a8 in Object.wait() [0x000000005bddf000]
   java.lang.Thread.State: TIMED_WAITING (on object monitor)
        at java.lang.Object.wait(Native Method)
        at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:142)
        - locked <0x00000000c3eb1a78> (a java.lang.ref.ReferenceQueue$Lock)
        at com.jsonde.profiler.heap.HeapAnalyzer.run(HeapAnalyzer.java:57)
        at java.lang.Thread.run(Thread.java:745)

"jSonde-daemon-thread-2" #25 daemon prio=5 os_prio=0 tid=0x0000000057bb5000 nid=0x744 in Object.wait() [0x000000005bb7e000]
   java.lang.Thread.State: WAITING (on object monitor)
        at java.lang.Object.wait(Native Method)
        at java.lang.Object.wait(Object.java:502)
        at com.jsonde.profiler.network.NetworkServerImpl.sendMessage(NetworkServerImpl.java:160)
        - locked <0x00000000c3ea0ae0> (a com.jsonde.profiler.network.NetworkServerImpl)
        at com.jsonde.profiler.ProfilerImpl.processMethodCall(ProfilerImpl.java:335)
        at com.jsonde.profiler.ThreadLocalProfiler.leaveMethodImpl(ThreadLocalProfiler.java:128)
        at com.jsonde.profiler.ProfilerImpl.leaveMethodImpl(ProfilerImpl.java:160)
        at com.jsonde.profiler.Profiler.leaveMethod(Profiler.java:132)
        at java.io.ObjectStreamClass$WeakClassKey.<init>(ObjectStreamClass.java:2308)
        at java.io.ObjectStreamClass.lookup(ObjectStreamClass.java:322)
        at java.io.ObjectOutputStream.writeObject0(ObjectOutputStream.java:1134)
        at java.io.ObjectOutputStream.writeObject(ObjectOutputStream.java:348)
        at com.jsonde.api.methodCall.MethodCallMessage.writeExternal(MethodCallMessage.java:107)
        at java.io.ObjectOutputStream.writeExternalData(ObjectOutputStream.java:1459)
        at java.io.ObjectOutputStream.writeOrdinaryObject(ObjectOutputStream.java:1430)
        at java.io.ObjectOutputStream.writeObject0(ObjectOutputStream.java:1178)
        at java.io.ObjectOutputStream.writeObject(ObjectOutputStream.java:348)
        at com.jsonde.profiler.network.ServerOutputWorker.run(ServerOutputWorker.java:49)
        at java.lang.Thread.run(Thread.java:745)

"jSonde-daemon-thread-1" #24 daemon prio=5 os_prio=0 tid=0x0000000057e2f000 nid=0x860 runnable [0x000000005ba1f000]
   java.lang.Thread.State: RUNNABLE
        at java.net.SocketInputStream.socketRead0(Native Method)
        at java.net.SocketInputStream.read(SocketInputStream.java:150)
        at java.net.SocketInputStream.read(SocketInputStream.java:121)
        at java.net.SocketInputStream.read(SocketInputStream.java:203)
        at java.io.ObjectInputStream$PeekInputStream.peek(ObjectInputStream.java:2296)
        at java.io.ObjectInputStream$BlockDataInputStream.peek(ObjectInputStream.java:2589)
        at java.io.ObjectInputStream$BlockDataInputStream.peekByte(ObjectInputStream.java:2599)
        at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1319)
        at java.io.ObjectInputStream.readObject(ObjectInputStream.java:371)
        at com.jsonde.profiler.network.ServerInputWorker.run(ServerInputWorker.java:49)
        at java.lang.Thread.run(Thread.java:745)

"Thread-3" #22 daemon prio=5 os_prio=0 tid=0x0000000057865800 nid=0x16d0 in Object.wait() [0x0000000059a8e000]
   java.lang.Thread.State: WAITING (on object monitor)
        at java.lang.Object.wait(Native Method)
        at java.lang.Object.wait(Object.java:502)
        at com.jsonde.profiler.network.NetworkServerImpl.sendMessage(NetworkServerImpl.java:160)
        - locked <0x00000000c3ea0ae0> (a com.jsonde.profiler.network.NetworkServerImpl)
        at com.jsonde.profiler.ProfilerImpl.registerClass(ProfilerImpl.java:250)
        at com.jsonde.instrumentation.TransformerCallback.getClassId(TransformerCallback.java:38)
        at com.jsonde.instrumentation.ClassTransformer.visit(ClassTransformer.java:66)
        at org.objectweb.asm.ClassReader.accept(Unknown Source)
        at org.objectweb.asm.ClassReader.accept(Unknown Source)
        at com.jsonde.instrumentation.ByteCodeTransformer.doTransform(ByteCodeTransformer.java:103)
        at com.jsonde.instrumentation.ByteCodeTransformer.transform(ByteCodeTransformer.java:43)
        at com.jsonde.agent.JSondeAgent.transform(JSondeAgent.java:264)
        at com.jsonde.agent.JSondeAgent.redefineLoadedClass(JSondeAgent.java:212)
        at com.jsonde.agent.JSondeAgent.redefineLoadedClasses(JSondeAgent.java:165)
        at com.jsonde.agent.JSondeAgent.access$000(JSondeAgent.java:23)
        at com.jsonde.agent.JSondeAgent$1.run(JSondeAgent.java:50)
        at java.lang.Thread.run(Thread.java:745)

"ajp-bio-8009-AsyncTimeout" #19 daemon prio=5 os_prio=0 tid=0x00000000587ea000 nid=0x119c waiting on condition [0x000000005b5be000]
   java.lang.Thread.State: TIMED_WAITING (sleeping)
        at java.lang.Thread.sleep(Native Method)
        at org.apache.tomcat.util.net.JIoEndpoint$AsyncTimeout.null (Redefined)
        at java.lang.Thread.run(Thread.java:745)

"ajp-bio-8009-Acceptor-0" #18 daemon prio=5 os_prio=0 tid=0x000000005827a800 nid=0x142c runnable [0x0000000059bcf000]
   java.lang.Thread.State: RUNNABLE
        at java.net.DualStackPlainSocketImpl.accept0(Native Method)
        at java.net.DualStackPlainSocketImpl.socketAccept(DualStackPlainSocketImpl.java:131)
        at java.net.AbstractPlainSocketImpl.accept(AbstractPlainSocketImpl.java:404)
        at java.net.PlainSocketImpl.accept(PlainSocketImpl.java:199)
        - locked <0x00000000c35488e0> (a java.net.SocksSocketImpl)
        at java.net.ServerSocket.implAccept(ServerSocket.java:545)
        at java.net.ServerSocket.accept(ServerSocket.java:513)
        at org.apache.tomcat.util.net.DefaultServerSocketFactory.null (Redefined)
        at org.apache.tomcat.util.net.JIoEndpoint$Acceptor.null (Redefined)
        at java.lang.Thread.run(Thread.java:745)

"http-bio-8080-AsyncTimeout" #17 daemon prio=5 os_prio=0 tid=0x000000005a0a0000 nid=0x126c waiting on condition [0x000000005b45f000]
   java.lang.Thread.State: TIMED_WAITING (sleeping)
        at java.lang.Thread.sleep(Native Method)
        at org.apache.tomcat.util.net.JIoEndpoint$AsyncTimeout.null (Redefined)
        at java.lang.Thread.run(Thread.java:745)

"http-bio-8080-Acceptor-0" #16 daemon prio=5 os_prio=0 tid=0x00000000587ec000 nid=0x137c in Object.wait() [0x000000005b22f000]
   java.lang.Thread.State: WAITING (on object monitor)
        at java.lang.Object.wait(Native Method)
        at java.lang.Object.wait(Object.java:502)
        at com.jsonde.profiler.network.NetworkServerImpl.sendMessage(NetworkServerImpl.java:160)
        - locked <0x00000000c3ea0ae0> (a com.jsonde.profiler.network.NetworkServerImpl)
        at com.jsonde.profiler.ProfilerImpl.describeClassImpl(ProfilerImpl.java:318)
        at com.jsonde.profiler.Profiler.describeClass(Profiler.java:117)
        at org.apache.tomcat.util.net.JIoEndpoint.setSocketOptions(JIoEndpoint.java)
        at org.apache.tomcat.util.net.JIoEndpoint$Acceptor.null (Redefined)
        at java.lang.Thread.run(Thread.java:745)

"ContainerBackgroundProcessor[StandardEngine[Catalina]]" #15 daemon prio=5 os_prio=0 tid=0x000000005a3e0800 nid=0x1540 in Object.wait() [0x000000005a02f000]
   java.lang.Thread.State: WAITING (on object monitor)
        at java.lang.Object.wait(Native Method)
        at java.lang.Object.wait(Object.java:502)
        at com.jsonde.profiler.network.NetworkServerImpl.sendMessage(NetworkServerImpl.java:160)
        - locked <0x00000000c3ea0ae0> (a com.jsonde.profiler.network.NetworkServerImpl)
        at com.jsonde.profiler.ProfilerImpl.processMethodCall(ProfilerImpl.java:335)
        at com.jsonde.profiler.ThreadLocalProfiler.leaveMethodImpl(ThreadLocalProfiler.java:128)
        at com.jsonde.profiler.ProfilerImpl.leaveMethodImpl(ProfilerImpl.java:160)
        at com.jsonde.profiler.Profiler.leaveMethod(Profiler.java:132)
        at org.apache.catalina.core.ContainerBase.access$100(ContainerBase.java:133)
        at org.apache.catalina.core.ContainerBase$ContainerBackgroundProcessor.null (Redefined)
        at java.lang.Thread.run(Thread.java:745)

"GC Daemon" #12 daemon prio=2 os_prio=-2 tid=0x0000000057bf1000 nid=0xfec in Object.wait() [0x000000005957f000]
   java.lang.Thread.State: TIMED_WAITING (on object monitor)
        at java.lang.Object.wait(Native Method)
        - waiting on <0x00000000c3294b88> (a sun.misc.GC$LatencyLock)
        at sun.misc.GC$Daemon.run(GC.java:117)
        - locked <0x00000000c3294b88> (a sun.misc.GC$LatencyLock)

"Service Thread" #9 daemon prio=9 os_prio=0 tid=0x0000000056bc8000 nid=0x1754 runnable [0x0000000000000000]
   java.lang.Thread.State: RUNNABLE

"C1 CompilerThread2" #8 daemon prio=9 os_prio=2 tid=0x000000005518f000 nid=0x113c waiting on condition [0x0000000000000000]
   java.lang.Thread.State: RUNNABLE

"C2 CompilerThread1" #7 daemon prio=9 os_prio=2 tid=0x000000005518d800 nid=0x870 waiting on condition [0x0000000000000000]
   java.lang.Thread.State: RUNNABLE

"C2 CompilerThread0" #6 daemon prio=9 os_prio=2 tid=0x0000000055185800 nid=0x1498 runnable [0x0000000000000000]
   java.lang.Thread.State: RUNNABLE

"Attach Listener" #5 daemon prio=5 os_prio=2 tid=0x0000000055184800 nid=0x1124 runnable [0x0000000000000000]
   java.lang.Thread.State: RUNNABLE

"Signal Dispatcher" #4 daemon prio=9 os_prio=2 tid=0x0000000055181000 nid=0x1264 waiting on condition [0x0000000000000000]
   java.lang.Thread.State: RUNNABLE

"Finalizer" #3 daemon prio=8 os_prio=1 tid=0x0000000055124000 nid=0x17e0 in Object.wait() [0x000000005667e000]
   java.lang.Thread.State: WAITING (on object monitor)
        at java.lang.Object.wait(Native Method)
        at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:142)
        - locked <0x00000000c32953f8> (a java.lang.ref.ReferenceQueue$Lock)
        at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:158)
        at java.lang.ref.Finalizer$FinalizerThread.run(Finalizer.java:209)

"Reference Handler" #2 daemon prio=10 os_prio=2 tid=0x000000005511b000 nid=0xd88 in Object.wait() [0x000000005654f000]
   java.lang.Thread.State: WAITING (on object monitor)
        at java.lang.Object.wait(Native Method)
        at java.lang.Object.wait(Object.java:502)
        at java.lang.ref.Reference$ReferenceHandler.run(Reference.java:157)
        - locked <0x00000000c3295438> (a java.lang.ref.Reference$Lock)

"main" #1 prio=5 os_prio=0 tid=0x0000000001ecf800 nid=0x17dc runnable [0x0000000002bbe000]
   java.lang.Thread.State: RUNNABLE
        at java.net.DualStackPlainSocketImpl.accept0(Native Method)
        at java.net.DualStackPlainSocketImpl.socketAccept(DualStackPlainSocketImpl.java:131)
        at java.net.AbstractPlainSocketImpl.accept(AbstractPlainSocketImpl.java:404)
        at java.net.PlainSocketImpl.accept(PlainSocketImpl.java:199)
        - locked <0x00000000c3eecd08> (a java.net.SocksSocketImpl)
        at java.net.ServerSocket.implAccept(ServerSocket.java:545)
        at java.net.ServerSocket.accept(ServerSocket.java:513)
        at org.apache.catalina.core.StandardServer.null (Redefined)
        at org.apache.catalina.startup.Catalina.null (Redefined)
        at org.apache.catalina.startup.Catalina.null (Redefined)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:483)
        at org.apache.catalina.startup.Bootstrap.null (Redefined)
        at org.apache.catalina.startup.Bootstrap.null (Redefined)

"VM Thread" os_prio=2 tid=0x0000000055115800 nid=0x13bc runnable

"GC task thread#0 (ParallelGC)" os_prio=0 tid=0x000000000272d000 nid=0x298 runnable

"GC task thread#1 (ParallelGC)" os_prio=0 tid=0x000000000272e800 nid=0xe4c runnable

"GC task thread#2 (ParallelGC)" os_prio=0 tid=0x0000000002730000 nid=0x1088 runnable

"GC task thread#3 (ParallelGC)" os_prio=0 tid=0x0000000002731800 nid=0x17a0 runnable

"VM Periodic Task Thread" os_prio=2 tid=0x0000000056bdb800 nid=0x1620 waiting on condition

JNI global references: 64

Heap
 PSYoungGen      total 70656K, used 52364K [0x00000000ebb80000, 0x00000000f4580000, 0x0000000100000000)
  eden space 63488K, 82% used [0x00000000ebb80000,0x00000000eeea3300,0x00000000ef980000)
  from space 7168K, 0% used [0x00000000ef980000,0x00000000ef980000,0x00000000f0080000)
  to   space 7168K, 0% used [0x00000000f3e80000,0x00000000f3e80000,0x00000000f4580000)
 ParOldGen       total 39424K, used 16742K [0x00000000c3200000, 0x00000000c5880000, 0x00000000ebb80000)
  object space 39424K, 42% used [0x00000000c3200000,0x00000000c42599e0,0x00000000c5880000)
 Metaspace       used 21255K, capacity 21576K, committed 21808K, reserved 1069056K
  class space    used 2026K, capacity 2142K, committed 2176K, reserved 1048576K

bedrin avatar May 24 '15 20:05 bedrin