federated icon indicating copy to clipboard operation
federated copied to clipboard

details = "`TensorFlowExecutor::CreateValue` can only create values for TensorFlow computations and intrinsics. Found computation of type 4"

Open karantai opened this issue 1 year ago • 3 comments

Hello,

I am trying to run the code from the Custom Federated Algorithms, Part 2: Implementing Federated Averaging but I get this error regarding gRPC.

`--------------------------------------------------------------------------- _InactiveRpcError Traceback (most recent call last) /home/giannisk/Desktop/giannis/MarineTraffic/Projects/Mobispaces/tensorflow_federated/CORE_TRAINING.py in line 115 112 losses.append(batch_loss(model, sample_batch)) 114 LOCAL_DATA_TYPE = tff.SequenceType(BATCH_TYPE) --> 115 locally_trained_model = local_train(initial_model, 0.1, federated_train_data[5])

File ~/anaconda3/envs/tensorflow_federated/lib/python3.10/site-packages/tensorflow_federated/python/core/impl/computation/computation_impl.py:148, in ConcreteComputation.call(self, *args, **kwargs) 142 def call(self, *args, **kwargs): 143 arg = function_utils.pack_args( 144 self._type_signature.parameter, # pytype: disable=attribute-error 145 args, 146 kwargs, 147 ) --> 148 return self._context_stack.current.invoke(self, arg)

File ~/anaconda3/envs/tensorflow_federated/lib/python3.10/site-packages/tensorflow_federated/python/core/impl/execution_contexts/sync_execution_context.py:65, in SyncExecutionContext.invoke(self, comp, arg) 64 def invoke(self, comp, arg): ---> 65 return self._async_runner.run_coro_and_return_result( 66 self._async_context.invoke(comp, arg) 67 )

File ~/anaconda3/envs/tensorflow_federated/lib/python3.10/site-packages/tensorflow_federated/python/common_libs/async_utils.py:149, in AsyncThreadRunner.run_coro_and_return_result(self, coro) 147 """Runs coroutine in the managed event loop, returning the result.""" 148 future = asyncio.run_coroutine_threadsafe(coro, self._event_loop) --> 149 return future.result()

File ~/anaconda3/envs/tensorflow_federated/lib/python3.10/concurrent/futures/_base.py:458, in Future.result(self, timeout) 456 raise CancelledError() 457 elif self._state == FINISHED: --> 458 return self.__get_result() 459 else: 460 raise TimeoutError()

File ~/anaconda3/envs/tensorflow_federated/lib/python3.10/concurrent/futures/_base.py:403, in Future.__get_result(self) 401 if self._exception: 402 try: --> 403 raise self._exception 404 finally: 405 # Break a reference cycle with the exception in self._exception 406 self = None

File ~/anaconda3/envs/tensorflow_federated/lib/python3.10/site-packages/tensorflow_federated/python/common_libs/retrying.py:119, in retry..retry_coro_fn(*args, **kwargs) 117 except Exception as e: # pylint: disable=broad-except 118 if not retry_on_exception_filter(e): --> 119 raise e 120 retry_wait_ms = min(wait_max_ms, retry_wait_ms * wait_multiplier) 121 # asyncio.sleep takes arguments in seconds.

File ~/anaconda3/envs/tensorflow_federated/lib/python3.10/site-packages/tensorflow_federated/python/common_libs/retrying.py:109, in retry..retry_coro_fn(*args, **kwargs) 107 while True: 108 try: --> 109 result = await fn(*args, **kwargs) 110 if retry_on_result_filter(result): 111 retry_wait_ms = min(wait_max_ms, retry_wait_ms * wait_multiplier)

File ~/anaconda3/envs/tensorflow_federated/lib/python3.10/site-packages/tensorflow_federated/python/core/impl/execution_contexts/async_execution_context.py:240, in AsyncExecutionContext.invoke(self, comp, arg) 235 if arg is not None: 236 arg = await tracing.wrap_coroutine_in_current_trace_context( 237 _ingest(executor, arg, comp.type_signature.parameter) 238 ) --> 240 return await tracing.wrap_coroutine_in_current_trace_context( 241 _invoke(executor, comp, arg, result_type) 242 )

File ~/anaconda3/envs/tensorflow_federated/lib/python3.10/site-packages/tensorflow_federated/python/common_libs/tracing.py:406, in wrap_coroutine_in_current_trace_context.._wrapped() 404 async def _wrapped(): 405 with _with_span_yields(trace_span_yields): --> 406 return await coro

File ~/anaconda3/envs/tensorflow_federated/lib/python3.10/site-packages/tensorflow_federated/python/core/impl/execution_contexts/async_execution_context.py:148, in _invoke(executor, comp, arg, result_type) 146 result = await executor.create_call(comp, arg) 147 py_typecheck.check_type(result, executor_value_base.ExecutorValue) --> 148 result_val = _unwrap(await result.compute()) 149 return type_conversions.type_to_py_container(result_val, result_type)

File ~/anaconda3/envs/tensorflow_federated/lib/python3.10/site-packages/tensorflow_federated/python/common_libs/tracing.py:207, in trace..async_trace(*fn_args, **fn_kwargs) 204 # Run the underlying function, recording the resulting value or exception 205 # and passing it back to the span generator 206 try: --> 207 result = await fn(*fn_args, **fn_kwargs) 208 completed = True 209 try:

File ~/anaconda3/envs/tensorflow_federated/lib/python3.10/site-packages/tensorflow_federated/python/core/impl/executors/remote_executor.py:85, in RemoteValue.compute(self) 83 @tracing.trace(span=True) 84 async def compute(self): ---> 85 return await self._executor._compute(self._value_ref, self._type_signature)

File ~/anaconda3/envs/tensorflow_federated/lib/python3.10/site-packages/tensorflow_federated/python/common_libs/tracing.py:207, in trace..async_trace(*fn_args, **fn_kwargs) 204 # Run the underlying function, recording the resulting value or exception 205 # and passing it back to the span generator 206 try: --> 207 result = await fn(*fn_args, **fn_kwargs) 208 completed = True 209 try:

File ~/anaconda3/envs/tensorflow_federated/lib/python3.10/site-packages/tensorflow_federated/python/core/impl/executors/remote_executor.py:333, in RemoteExecutor._compute(self, value_ref, type_spec) 328 return await self._compute_stream_structs(value_ref, type_spec) 330 request = executor_pb2.ComputeRequest( 331 executor=self._executor_id, value_ref=value_ref 332 ) --> 333 response = self._stub.compute(request) 334 py_typecheck.check_type(response, executor_pb2.ComputeResponse) 335 value, _ = value_serialization.deserialize_value(response.value, type_spec)

File ~/anaconda3/envs/tensorflow_federated/lib/python3.10/site-packages/tensorflow_federated/python/core/impl/executors/remote_executor_grpc_stub.py:99, in RemoteExecutorGrpcStub.compute(self, request) 95 def compute( 96 self, request: executor_pb2.ComputeRequest 97 ) -> executor_pb2.ComputeResponse: 98 """Dispatches a Compute gRPC.""" ---> 99 return _request(self._stub.Compute, request)

File ~/anaconda3/envs/tensorflow_federated/lib/python3.10/site-packages/tensorflow_federated/python/common_libs/tracing.py:236, in trace..sync_trace(*fn_args, **fn_kwargs) 234 completed = False 235 try: --> 236 result = fn(*fn_args, **fn_kwargs) 237 completed = True 238 try:

File ~/anaconda3/envs/tensorflow_federated/lib/python3.10/site-packages/tensorflow_federated/python/core/impl/executors/remote_executor_grpc_stub.py:40, in _request(rpc_func, request) 38 raise executors_errors.RetryableGRPCError(e) 39 else: ---> 40 raise e

File ~/anaconda3/envs/tensorflow_federated/lib/python3.10/site-packages/tensorflow_federated/python/core/impl/executors/remote_executor_grpc_stub.py:31, in _request(rpc_func, request) 29 with tracing.wrap_rpc_in_trace_context(): 30 try: ---> 31 return rpc_func(request) 32 except grpc.RpcError as e: 33 if ( 34 isinstance(e, grpc.Call) 35 and e.code() in executors_errors.get_grpc_retryable_error_codes() 36 ):

File ~/anaconda3/envs/tensorflow_federated/lib/python3.10/site-packages/grpc/_channel.py:946, in _UnaryUnaryMultiCallable.call(self, request, timeout, metadata, credentials, wait_for_ready, compression) 937 def call(self, 938 request, 939 timeout=None, (...) 942 wait_for_ready=None, 943 compression=None): 944 state, call, = self._blocking(request, timeout, metadata, credentials, 945 wait_for_ready, compression) --> 946 return _end_unary_response_blocking(state, call, False, None)

File ~/anaconda3/envs/tensorflow_federated/lib/python3.10/site-packages/grpc/_channel.py:849, in _end_unary_response_blocking(state, call, with_call, deadline) 847 return state.response 848 else: --> 849 raise _InactiveRpcError(state)

_InactiveRpcError: <_InactiveRpcError of RPC that terminated with: status = StatusCode.INVALID_ARGUMENT details = "TensorFlowExecutor::CreateValue can only create values for TensorFlow computations and intrinsics. Found computation of type 4" debug_error_string = "UNKNOWN:Error received from peer ipv4:127.0.0.1:36631 {grpc_message:"TensorFlowExecutor::CreateValue can only create values for TensorFlow computations and intrinsics. Found computation of type 4", grpc_status:3, created_time:"2023-10-04T14:09:31.41726913+03:00"}"

`

I am using python 3.10.2, tensorflow_federated 0.61.0 and tensorflow 2.12.1

karantai avatar Oct 04 '23 11:10 karantai