ERROR:tensorflow:Model diverged with loss = NaN.

[humf]

INFO:tensorflow:Using default config. WARNING:tensorflow:Using temporary folder as model directory: /tmp/tmpzXWlhI INFO:tensorflow:Using config: {'_save_checkpoints_secs': 600, '_session_config': None, '_keep_checkpoint_max': 5, '_task_type': 'worker', '_global_id_in_cluster': 0, '_is_chief': True, '_cluster_spec': <tensorflow.python.training.server_lib.ClusterSpec object at 0x7fc8d7933650>, '_evaluation_master': '', '_save_checkpoints_steps': None, '_keep_checkpoint_every_n_hours': 10000, '_service': None, '_num_ps_replicas': 0, '_tf_random_seed': None, '_master': '', '_device_fn': None, '_num_worker_replicas': 1, '_task_id': 0, '_log_step_count_steps': 100, '_model_dir': '/tmp/tmpzXWlhI', '_train_distribute': None, '_save_summary_steps': 100} INFO:tensorflow:Calling model_fn. INFO:tensorflow:Done calling model_fn. INFO:tensorflow:Create CheckpointSaverHook. INFO:tensorflow:Graph was finalized. INFO:tensorflow:Running local_init_op. INFO:tensorflow:Done running local_init_op. INFO:tensorflow:Saving checkpoints for 0 into /tmp/tmpzXWlhI/model.ckpt. ERROR:tensorflow:Model diverged with loss = NaN.

NanLossDuringTrainingErrorTraceback (most recent call last) in () 32 optimizer=tf.train.AdamOptimizer(0.001)) 33 ---> 34 estimator.train(input_fn=train_input_fn, steps=200) 35 evaluation_input_fn = tf.contrib.timeseries.WholeDatasetInputFn(reader) 36 evaluation = estimator.evaluate(input_fn=evaluation_input_fn, steps=1)

/usr/local/anaconda2/lib/python2.7/site-packages/tensorflow/python/estimator/estimator.pyc in train(self, input_fn, hooks, steps, max_steps, saving_listeners) 364 365 saving_listeners = _check_listeners_type(saving_listeners) --> 366 loss = self._train_model(input_fn, hooks, saving_listeners) 367 logging.info('Loss for final step: %s.', loss) 368 return self

/usr/local/anaconda2/lib/python2.7/site-packages/tensorflow/python/estimator/estimator.pyc in _train_model(self, input_fn, hooks, saving_listeners) 1117 return self._train_model_distributed(input_fn, hooks, saving_listeners) 1118 else: -> 1119 return self._train_model_default(input_fn, hooks, saving_listeners) 1120 1121 def _train_model_default(self, input_fn, hooks, saving_listeners):

/usr/local/anaconda2/lib/python2.7/site-packages/tensorflow/python/estimator/estimator.pyc in _train_model_default(self, input_fn, hooks, saving_listeners) 1133 return self._train_with_estimator_spec(estimator_spec, worker_hooks, 1134 hooks, global_step_tensor, -> 1135 saving_listeners) 1136 1137 def _train_model_distributed(self, input_fn, hooks, saving_listeners):

/usr/local/anaconda2/lib/python2.7/site-packages/tensorflow/python/estimator/estimator.pyc in _train_with_estimator_spec(self, estimator_spec, worker_hooks, hooks, global_step_tensor, saving_listeners) 1334 loss = None 1335 while not mon_sess.should_stop(): -> 1336 _, loss = mon_sess.run([estimator_spec.train_op, estimator_spec.loss]) 1337 return loss 1338

/usr/local/anaconda2/lib/python2.7/site-packages/tensorflow/python/training/monitored_session.pyc in run(self, fetches, feed_dict, options, run_metadata) 575 feed_dict=feed_dict, 576 options=options, --> 577 run_metadata=run_metadata) 578 579 def run_step_fn(self, step_fn):

/usr/local/anaconda2/lib/python2.7/site-packages/tensorflow/python/training/monitored_session.pyc in run(self, fetches, feed_dict, options, run_metadata) 1051 feed_dict=feed_dict, 1052 options=options, -> 1053 run_metadata=run_metadata) 1054 except _PREEMPTION_ERRORS as e: 1055 logging.info('An error was raised. This may be due to a preemption in '

/usr/local/anaconda2/lib/python2.7/site-packages/tensorflow/python/training/monitored_session.pyc in run(self, *args, **kwargs) 1142 raise six.reraise(*original_exc_info) 1143 else: -> 1144 raise six.reraise(*original_exc_info) 1145 1146

/usr/local/anaconda2/lib/python2.7/site-packages/tensorflow/python/training/monitored_session.pyc in run(self, *args, **kwargs) 1127 def run(self, *args, **kwargs): 1128 try: -> 1129 return self._sess.run(*args, **kwargs) 1130 except _PREEMPTION_ERRORS: 1131 raise

/usr/local/anaconda2/lib/python2.7/site-packages/tensorflow/python/training/monitored_session.pyc in run(self, fetches, feed_dict, options, run_metadata) 1207 results=outputs[hook] if hook in outputs else None, 1208 options=options, -> 1209 run_metadata=run_metadata)) 1210 self._should_stop = self._should_stop or run_context.stop_requested 1211

/usr/local/anaconda2/lib/python2.7/site-packages/tensorflow/python/training/basic_session_run_hooks.pyc in after_run(self, run_context, run_values) 633 if self._fail_on_nan_loss: 634 logging.error(failure_message) --> 635 raise NanLossDuringTrainingError 636 else: 637 logging.warning(failure_message)

NanLossDuringTrainingError: NaN loss during training.