InvalidArgumentError: Nan in summary histogram for: hidden/kernel_0 [Op:WriteHistogramSummary]

[circle2nd]

[17_deep_learning]
02_how_to_use_tensorflow
6 Train Model

when run the code:

training=model.fit(X,
Y, epochs=50, validation_split=.2, batch_size=128, verbose=1, callbacks=[tb_callback])

encountered an error：

Epoch 1/50 2/313 [..............................] - ETA: 1:32 - loss: nan - accuracy: 0.4648WARNING:tensorflow:Callbacks method on_train_batch_begin is slow compared to the batch time (batch time: 0.0020s vs on_train_batch_begin time: 0.0070s). Check your callbacks. WARNING:tensorflow:Callbacks method on_train_batch_end is slow compared to the batch time (batch time: 0.0020s vs on_train_batch_end time: 0.5843s). Check your callbacks. 306/313 [============================>.] - ETA: 0s - loss: nan - accuracy: 0.4987

InvalidArgumentError Traceback (most recent call last) Input In [25], in <cell line: 1>() ----> 1 training=model.fit(X,
2 Y, 3 epochs=50, 4 validation_split=.2, 5 batch_size=128, 6 verbose=1, 7 callbacks=[tb_callback])

File F:\Python\Miniconda3\envs\ml4t\lib\site-packages\tensorflow\python\keras\engine\training.py:108, in enable_multi_worker.._method_wrapper(self, *args, **kwargs) 106 def _method_wrapper(self, *args, **kwargs): 107 if not self._in_multi_worker_mode(): # pylint: disable=protected-access --> 108 return method(self, *args, **kwargs) 110 # Running inside run_distribute_coordinator already. 111 if dc_context.get_current_worker_context():

File F:\Python\Miniconda3\envs\ml4t\lib\site-packages\tensorflow\python\keras\engine\training.py:1137, in Model.fit(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, validation_batch_size, validation_freq, max_queue_size, workers, use_multiprocessing) 1134 val_logs = {'val_' + name: val for name, val in val_logs.items()} 1135 epoch_logs.update(val_logs) -> 1137 callbacks.on_epoch_end(epoch, epoch_logs) 1138 training_logs = epoch_logs 1139 if self.stop_training:

File F:\Python\Miniconda3\envs\ml4t\lib\site-packages\tensorflow\python\keras\callbacks.py:412, in CallbackList.on_epoch_end(self, epoch, logs) 410 for callback in self.callbacks: 411 if getattr(callback, '_supports_tf_logs', False): --> 412 callback.on_epoch_end(epoch, logs) 413 else: 414 if numpy_logs is None: # Only convert once.

File F:\Python\Miniconda3\envs\ml4t\lib\site-packages\tensorflow\python\keras\callbacks.py:2182, in TensorBoard.on_epoch_end(self, epoch, logs) 2179 self._log_epoch_metrics(epoch, logs) 2181 if self.histogram_freq and epoch % self.histogram_freq == 0: -> 2182 self._log_weights(epoch) 2184 if self.embeddings_freq and epoch % self.embeddings_freq == 0: 2185 self._log_embeddings(epoch)

File F:\Python\Miniconda3\envs\ml4t\lib\site-packages\tensorflow\python\keras\callbacks.py:2234, in TensorBoard.log_weights(self, epoch) 2232 for weight in layer.weights: 2233 weight_name = weight.name.replace(':', '') -> 2234 summary_ops_v2.histogram(weight_name, weight, step=epoch) 2235 if self.write_images: 2236 self._log_weight_as_image(weight, weight_name, epoch)

File F:\Python\Miniconda3\envs\ml4t\lib\site-packages\tensorflow\python\ops\summary_ops_v2.py:836, in histogram(name, tensor, family, step) 827 def function(tag, scope): 828 # Note the identity to move the tensor to the CPU. 829 return gen_summary_ops.write_histogram_summary( 830 _summary_state.writer._resource, # pylint: disable=protected-access 831 _choose_step(step), 832 tag, 833 array_ops.identity(tensor), 834 name=scope) --> 836 return summary_writer_function(name, tensor, function, family=family)

File F:\Python\Miniconda3\envs\ml4t\lib\site-packages\tensorflow\python\ops\summary_ops_v2.py:764, in summary_writer_function(name, tensor, function, family) 762 return control_flow_ops.no_op() 763 with ops.device("cpu:0"): --> 764 op = smart_cond.smart_cond( 765 should_record_summaries(), record, _nothing, name="") 766 if not context.executing_eagerly(): 767 ops.add_to_collection(ops.GraphKeys._SUMMARY_COLLECTION, op) # pylint: disable=protected-access

File F:\Python\Miniconda3\envs\ml4t\lib\site-packages\tensorflow\python\framework\smart_cond.py:54, in smart_cond(pred, true_fn, false_fn, name) 52 if pred_value is not None: 53 if pred_value: ---> 54 return true_fn() 55 else: 56 return false_fn()

File F:\Python\Miniconda3\envs\ml4t\lib\site-packages\tensorflow\python\ops\summary_ops_v2.py:758, in summary_writer_function..record() 755 def record(): 756 with ops.name_scope(name_scope), summary_op_util.summary_scope( 757 name, family, values=[tensor]) as (tag, scope): --> 758 with ops.control_dependencies([function(tag, scope)]): 759 return constant_op.constant(True)

File F:\Python\Miniconda3\envs\ml4t\lib\site-packages\tensorflow\python\ops\summary_ops_v2.py:829, in histogram..function(tag, scope) 827 def function(tag, scope): 828 # Note the identity to move the tensor to the CPU. --> 829 return gen_summary_ops.write_histogram_summary( 830 _summary_state.writer._resource, # pylint: disable=protected-access 831 _choose_step(step), 832 tag, 833 array_ops.identity(tensor), 834 name=scope)

File F:\Python\Miniconda3\envs\ml4t\lib\site-packages\tensorflow\python\ops\gen_summary_ops.py:478, in write_histogram_summary(writer, step, tag, values, name) 476 pass 477 try: --> 478 return write_histogram_summary_eager_fallback( 479 writer, step, tag, values, name=name, ctx=_ctx) 480 except _core._SymbolicException: 481 pass # Add nodes to the TensorFlow graph.

File F:\Python\Miniconda3\envs\ml4t\lib\site-packages\tensorflow\python\ops\gen_summary_ops.py:497, in write_histogram_summary_eager_fallback(writer, step, tag, values, name, ctx) 495 _inputs_flat = [writer, step, tag, values] 496 _attrs = ("T", _attr_T) --> 497 _result = _execute.execute(b"WriteHistogramSummary", 0, inputs=_inputs_flat, 498 attrs=_attrs, ctx=ctx, name=name) 499 _result = None 500 return _result

File F:\Python\Miniconda3\envs\ml4t\lib\site-packages\tensorflow\python\eager\execute.py:59, in quick_execute(op_name, num_outputs, inputs, attrs, ctx, name) 57 try: 58 ctx.ensure_initialized() ---> 59 tensors = pywrap_tfe.TFE_Py_Execute(ctx._handle, device_name, op_name, 60 inputs, attrs, num_outputs) 61 except core._NotOkStatusException as e: 62 if name is not None:

InvalidArgumentError: Nan in summary histogram for: hidden/kernel_0 [Op:WriteHistogramSummary]