toy异常——<_Rendezvous of RPC that terminated with: status = StatusCode.INTERNAL

[holdbelief]

toy test job 202208221609324003940 is running toy test job 202208221609324003940 is running toy test job 202208221609324003940 is running toy test job 202208221609324003940 is running check job status timeout [ERROR] [2022-08-22 16:12:38,207] [202208221609324003940] [491526:281457369760448] - [task_executor.run] [line:243]: ('Failed to call command: CommandURI(_uri=v1/cluster-manager/session/getOrCreateSession) to endpoint: 172.20.13.60:4670, caused by: ', <_Rendezvous of RPC that terminated with: status = StatusCode.INTERNAL details = "172.20.13.60:4670: java.lang.reflect.InvocationTargetException at sun.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at com.webank.eggroll.core.command.CommandRouter$.dispatch(CommandRouter.scala:155) at com.webank.eggroll.core.command.CommandService.com$webank$eggroll$core$command$CommandService$$run$body$1(CommandService.scala:47) at com.webank.eggroll.core.command.CommandService$$anonfun$1.run(CommandService.scala:41) at com.webank.eggroll.core.grpc.server.GrpcServerWrapper.wrapGrpcServerRunnable(GrpcServerWrapper.java:43) at com.webank.eggroll.core.command.CommandService.call(CommandService.scala:41) at com.webank.eggroll.core.command.CommandServiceGrpc$MethodHandlers.invoke(CommandServiceGrpc.java:217) at io.grpc.stub.ServerCalls$UnaryServerCallHandler$UnaryServerCallListener.onHalfClose(ServerCalls.java:172) at io.grpc.internal.ServerCallImpl$ServerStreamListenerImpl.halfClosed(ServerCallImpl.java:331) at io.grpc.internal.ServerImpl$JumpToApplicationThreadServerStreamListener$1HalfClosed.runInContext(ServerImpl.java:817) at io.grpc.internal.ContextRunnable.run(ContextRunnable.java:37) at io.grpc.internal.SerializingExecutor.run(SerializingExecutor.java:123) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) Caused by: java.lang.NullPointerException at com.webank.eggroll.core.resourcemanager.SessionManagerService.getOrCreateSession(SessionManager.scala:89) ... 17 more " debug_error_string = "{"created":"@1661155958.206007866","description":"Error received from peer ipv4:172.20.13.60:4670","file":"src/core/lib/surface/call.cc","file_line":1055,"grpc_message":"172.20.13.60:4670: java.lang.reflect.InvocationTargetException\n\tat sun.reflect.GeneratedMethodAccessor3.invoke(Unknown Source)\n\tat sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\n\tat java.lang.reflect.Method.invoke(Method.java:498)\n\tat com.webank.eggroll.core.command.CommandRouter$.dispatch(CommandRouter.scala:155)\n\tat com.webank.eggroll.core.command.CommandService.com$webank$eggroll$core$command$CommandService$$run$body$1(CommandService.scala:47)\n\tat com.webank.eggroll.core.command.CommandService$$anonfun$1.run(CommandService.scala:41)\n\tat com.webank.eggroll.core.grpc.server.GrpcServerWrapper.wrapGrpcServerRunnable(GrpcServerWrapper.java:43)\n\tat com.webank.eggroll.core.command.CommandService.call(CommandService.scala:41)\n\tat com.webank.eggroll.core.command.CommandServiceGrpc$MethodHandlers.invoke(CommandServiceGrpc.java:217)\n\tat io.grpc.stub.ServerCalls$UnaryServerCallHandler$UnaryServerCallListener.onHalfClose(ServerCalls.java:172)\n\tat io.grpc.internal.ServerCallImpl$ServerStreamListenerImpl.halfClosed(ServerCallImpl.java:331)\n\tat io.grpc.internal.ServerImpl$JumpToApplicationThreadServerStreamListener$1HalfClosed.runInContext(ServerImpl.java:817)\n\tat io.grpc.internal.ContextRunnable.run(ContextRunnable.java:37)\n\tat io.grpc.internal.SerializingExecutor.run(SerializingExecutor.java:123)\n\tat java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)\n\tat java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)\n\tat java.lang.Thread.run(Thread.java:750)\nCaused by: java.lang.NullPointerException\n\tat com.webank.eggroll.core.resourcemanager.SessionManagerService.getOrCreateSession(SessionManager.scala:89)\n\t... 17 more\n","grpc_status":13}"

) Traceback (most recent call last): File "/data/projects/fate/eggroll/python/eggroll/core/client.py", line 84, in sync_send response = _command_stub.call(request.to_proto()) File "/data/projects/fate/common/python/venv/lib/python3.7/site-packages/grpc/_channel.py", line 604, in call return _end_unary_response_blocking(state, call, False, None) File "/data/projects/fate/common/python/venv/lib/python3.7/site-packages/grpc/_channel.py", line 506, in _end_unary_response_blocking raise _Rendezvous(state, None, None, deadline) grpc._channel._Rendezvous: <_Rendezvous of RPC that terminated with: status = StatusCode.INTERNAL details = "172.20.13.60:4670: java.lang.reflect.InvocationTargetException at sun.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at com.webank.eggroll.core.command.CommandRouter$.dispatch(CommandRouter.scala:155) at com.webank.eggroll.core.command.CommandService.com$webank$eggroll$core$command$CommandService$$run$body$1(CommandService.scala:47) at com.webank.eggroll.core.command.CommandService$$anonfun$1.run(CommandService.scala:41) at com.webank.eggroll.core.grpc.server.GrpcServerWrapper.wrapGrpcServerRunnable(GrpcServerWrapper.java:43) at com.webank.eggroll.core.command.CommandService.call(CommandService.scala:41) at com.webank.eggroll.core.command.CommandServiceGrpc$MethodHandlers.invoke(CommandServiceGrpc.java:217) at io.grpc.stub.ServerCalls$UnaryServerCallHandler$UnaryServerCallListener.onHalfClose(ServerCalls.java:172) at io.grpc.internal.ServerCallImpl$ServerStreamListenerImpl.halfClosed(ServerCallImpl.java:331) at io.grpc.internal.ServerImpl$JumpToApplicationThreadServerStreamListener$1HalfClosed.runInContext(ServerImpl.java:817) at io.grpc.internal.ContextRunnable.run(ContextRunnable.java:37) at io.grpc.internal.SerializingExecutor.run(SerializingExecutor.java:123) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) Caused by: java.lang.NullPointerException at com.webank.eggroll.core.resourcemanager.SessionManagerService.getOrCreateSession(SessionManager.scala:89) ... 17 more " debug_error_string = "{"created":"@1661155958.206007866","description":"Error received from peer ipv4:172.20.13.60:4670","file":"src/core/lib/surface/call.cc","file_line":1055,"grpc_message":"172.20.13.60:4670: java.lang.reflect.InvocationTargetException\n\tat sun.reflect.GeneratedMethodAccessor3.invoke(Unknown Source)\n\tat sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\n\tat java.lang.reflect.Method.invoke(Method.java:498)\n\tat com.webank.eggroll.core.command.CommandRouter$.dispatch(CommandRouter.scala:155)\n\tat com.webank.eggroll.core.command.CommandService.com$webank$eggroll$core$command$CommandService$$run$body$1(CommandService.scala:47)\n\tat com.webank.eggroll.core.command.CommandService$$anonfun$1.run(CommandService.scala:41)\n\tat com.webank.eggroll.core.grpc.server.GrpcServerWrapper.wrapGrpcServerRunnable(GrpcServerWrapper.java:43)\n\tat com.webank.eggroll.core.command.CommandService.call(CommandService.scala:41)\n\tat com.webank.eggroll.core.command.CommandServiceGrpc$MethodHandlers.invoke(CommandServiceGrpc.java:217)\n\tat io.grpc.stub.ServerCalls$UnaryServerCallHandler$UnaryServerCallListener.onHalfClose(ServerCalls.java:172)\n\tat io.grpc.internal.ServerCallImpl$ServerStreamListenerImpl.halfClosed(ServerCallImpl.java:331)\n\tat io.grpc.internal.ServerImpl$JumpToApplicationThreadServerStreamListener$1HalfClosed.runInContext(ServerImpl.java:817)\n\tat io.grpc.internal.ContextRunnable.run(ContextRunnable.java:37)\n\tat io.grpc.internal.SerializingExecutor.run(SerializingExecutor.java:123)\n\tat java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)\n\tat java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)\n\tat java.lang.Thread.run(Thread.java:750)\nCaused by: java.lang.NullPointerException\n\tat com.webank.eggroll.core.resourcemanager.SessionManagerService.getOrCreateSession(SessionManager.scala:89)\n\t... 17 more\n","grpc_status":13}"

During handling of the above exception, another exception occurred:

Traceback (most recent call last): File "/data/projects/fate/fateflow/python/fate_flow/worker/task_executor.py", line 134, in run sess.init_computing(computing_session_id=args.session_id, options=session_options) File "/data/projects/fate/fate/python/fate_arch/session/_session.py", line 116, in init_computing session_id=computing_session_id, options=options File "/data/projects/fate/fate/python/fate_arch/computing/eggroll/_csession.py", line 36, in init self._rp_session = session_init(session_id=session_id, options=options) File "/data/projects/fate/eggroll/python/eggroll/core/session.py", line 42, in session_init er_session = ErSession(session_id=session_id, options=options) File "/data/projects/fate/eggroll/python/eggroll/core/session.py", line 199, in init self.__session_meta = self._cluster_manager_client.get_or_create_session(session_meta) File "/data/projects/fate/eggroll/python/eggroll/core/client.py", line 189, in get_or_create_session serdes_type=self.__serdes_type)) File "/data/projects/fate/eggroll/python/eggroll/core/client.py", line 239, in __do_sync_request_internal serdes_type=serdes_type) File "/data/projects/fate/eggroll/python/eggroll/core/client.py", line 67, in simple_sync_send results = self.sync_send(inputs=[input], output_types=[output_type], endpoint=endpoint, command_uri=command_uri, serdes_type=serdes_type) File "/data/projects/fate/eggroll/python/eggroll/core/client.py", line 97, in sync_send raise CommandCallError(command_uri, endpoint, e) eggroll.core.client.CommandCallError: ('Failed to call command: CommandURI(_uri=v1/cluster-manager/session/getOrCreateSession) to endpoint: 172.20.13.60:4670, caused by: ', <_Rendezvous of RPC that terminated with: status = StatusCode.INTERNAL details = "172.20.13.60:4670: java.lang.reflect.InvocationTargetException at sun.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at com.webank.eggroll.core.command.CommandRouter$.dispatch(CommandRouter.scala:155) at com.webank.eggroll.core.command.CommandService.com$webank$eggroll$core$command$CommandService$$run$body$1(CommandService.scala:47) at com.webank.eggroll.core.command.CommandService$$anonfun$1.run(CommandService.scala:41) at com.webank.eggroll.core.grpc.server.GrpcServerWrapper.wrapGrpcServerRunnable(GrpcServerWrapper.java:43) at com.webank.eggroll.core.command.CommandService.call(CommandService.scala:41) at com.webank.eggroll.core.command.CommandServiceGrpc$MethodHandlers.invoke(CommandServiceGrpc.java:217) at io.grpc.stub.ServerCalls$UnaryServerCallHandler$UnaryServerCallListener.onHalfClose(ServerCalls.java:172) at io.grpc.internal.ServerCallImpl$ServerStreamListenerImpl.halfClosed(ServerCallImpl.java:331) at io.grpc.internal.ServerImpl$JumpToApplicationThreadServerStreamListener$1HalfClosed.runInContext(ServerImpl.java:817) at io.grpc.internal.ContextRunnable.run(ContextRunnable.java:37) at io.grpc.internal.SerializingExecutor.run(SerializingExecutor.java:123) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) Caused by: java.lang.NullPointerException at com.webank.eggroll.core.resourcemanager.SessionManagerService.getOrCreateSession(SessionManager.scala:89) ... 17 more " debug_error_string = "{"created":"@1661155958.206007866","description":"Error received from peer ipv4:172.20.13.60:4670","file":"src/core/lib/surface/call.cc","file_line":1055,"grpc_message":"172.20.13.60:4670: java.lang.reflect.InvocationTargetException\n\tat sun.reflect.GeneratedMethodAccessor3.invoke(Unknown Source)\n\tat sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\n\tat java.lang.reflect.Method.invoke(Method.java:498)\n\tat com.webank.eggroll.core.command.CommandRouter$.dispatch(CommandRouter.scala:155)\n\tat com.webank.eggroll.core.command.CommandService.com$webank$eggroll$core$command$CommandService$$run$body$1(CommandService.scala:47)\n\tat com.webank.eggroll.core.command.CommandService$$anonfun$1.run(CommandService.scala:41)\n\tat com.webank.eggroll.core.grpc.server.GrpcServerWrapper.wrapGrpcServerRunnable(GrpcServerWrapper.java:43)\n\tat com.webank.eggroll.core.command.CommandService.call(CommandService.scala:41)\n\tat com.webank.eggroll.core.command.CommandServiceGrpc$MethodHandlers.invoke(CommandServiceGrpc.java:217)\n\tat io.grpc.stub.ServerCalls$UnaryServerCallHandler$UnaryServerCallListener.onHalfClose(ServerCalls.java:172)\n\tat io.grpc.internal.ServerCallImpl$ServerStreamListenerImpl.halfClosed(ServerCallImpl.java:331)\n\tat io.grpc.internal.ServerImpl$JumpToApplicationThreadServerStreamListener$1HalfClosed.runInContext(ServerImpl.java:817)\n\tat io.grpc.internal.ContextRunnable.run(ContextRunnable.java:37)\n\tat io.grpc.internal.SerializingExecutor.run(SerializingExecutor.java:123)\n\tat java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)\n\tat java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)\n\tat java.lang.Thread.run(Thread.java:750)\nCaused by: java.lang.NullPointerException\n\tat com.webank.eggroll.core.resourcemanager.SessionManagerService.getOrCreateSession(SessionManager.scala:89)\n\t... 17 more\n","grpc_status":13}"

)