issue during training the data

[karthika-devi]

I got the below error after 1220 steps of training get finished during running the udc_train.py file

Traceback (most recent call last): File "/usr/local/lib/python3.4/dist-packages/tensorflow/python/client/session.py", line 1021, in _do_call return fn(*args) File "/usr/local/lib/python3.4/dist-packages/tensorflow/python/client/session.py", line 1003, in _run_fn status, run_metadata) File "/usr/lib/python3.4/contextlib.py", line 66, in exit next(self.gen) File "/usr/local/lib/python3.4/dist-packages/tensorflow/python/framework/errors_impl.py", line 469, in raise_exception_on_not_ok_status pywrap_tensorflow.TF_GetCode(status)) tensorflow.python.framework.errors_impl.InvalidArgumentError: Incompatible shapes: [80,1] vs. [160,1] [[Node: prediction/logistic_loss/mul = Mul[T=DT_FLOAT, _device="/job:localhost/replica:0/task:0/cpu:0"](prediction/Squeeze, prediction/ToFloat)]]

During handling of the above exception, another exception occurred:

Traceback (most recent call last): File "udc_train.py", line 64, in tf.app.run() File "/usr/local/lib/python3.4/dist-packages/tensorflow/python/platform/app.py", line 43, in run sys.exit(main(sys.argv[:1] + flags_passthrough)) File "udc_train.py", line 61, in main estimator.fit(input_fn=input_fn_train, steps=None, monitors=[eval_monitor]) File "/usr/local/lib/python3.4/dist-packages/tensorflow/python/util/deprecation.py", line 191, in new_func return func(*args, **kwargs) File "/usr/local/lib/python3.4/dist-packages/tensorflow/contrib/learn/python/learn/estimators/estimator.py", line 355, in fit max_steps=max_steps) File "/usr/local/lib/python3.4/dist-packages/tensorflow/contrib/learn/python/learn/estimators/estimator.py", line 733, in _train_model max_steps=max_steps) File "/usr/local/lib/python3.4/dist-packages/tensorflow/contrib/learn/python/learn/graph_actions.py", line 301, in _monitored_train None) File "/usr/local/lib/python3.4/dist-packages/tensorflow/python/training/monitored_session.py", line 473, in run run_metadata=run_metadata) File "/usr/local/lib/python3.4/dist-packages/tensorflow/python/training/monitored_session.py", line 628, in run run_metadata=run_metadata) File "/usr/local/lib/python3.4/dist-packages/tensorflow/python/training/monitored_session.py", line 595, in run return self._sess.run(*args, **kwargs) File "/usr/local/lib/python3.4/dist-packages/tensorflow/python/training/monitored_session.py", line 737, in run run_metadata=run_metadata)) File "/usr/local/lib/python3.4/dist-packages/tensorflow/contrib/learn/python/learn/monitors.py", line 1210, in after_run induce_stop = m.step_end(self._last_step, result) File "/usr/local/lib/python3.4/dist-packages/tensorflow/contrib/learn/python/learn/monitors.py", line 409, in step_end return self.every_n_step_end(step, output) File "/usr/local/lib/python3.4/dist-packages/tensorflow/contrib/learn/python/learn/monitors.py", line 710, in every_n_step_end steps=self.eval_steps, metrics=self.metrics, name=self.name) File "/usr/local/lib/python3.4/dist-packages/tensorflow/python/util/deprecation.py", line 191, in new_func return func(*args, **kwargs) File "/usr/local/lib/python3.4/dist-packages/tensorflow/contrib/learn/python/learn/estimators/estimator.py", line 427, in evaluate name=name) File "/usr/local/lib/python3.4/dist-packages/tensorflow/contrib/learn/python/learn/estimators/estimator.py", line 809, in _evaluate_model max_steps=steps) File "/usr/local/lib/python3.4/dist-packages/tensorflow/contrib/learn/python/learn/graph_actions.py", line 754, in evaluate session.run(update_op, feed_dict=feed_dict) File "/usr/local/lib/python3.4/dist-packages/tensorflow/python/client/session.py", line 766, in run run_metadata_ptr) File "/usr/local/lib/python3.4/dist-packages/tensorflow/python/client/session.py", line 964, in _run feed_dict_string, options, run_metadata) File "/usr/local/lib/python3.4/dist-packages/tensorflow/python/client/session.py", line 1014, in _do_run target_list, options, run_metadata) File "/usr/local/lib/python3.4/dist-packages/tensorflow/python/client/session.py", line 1034, in _do_call raise type(e)(node_def, op, message) tensorflow.python.framework.errors_impl.InvalidArgumentError: Incompatible shapes: [80,1] vs. [160,1] [[Node: prediction/logistic_loss/mul = Mul[T=DT_FLOAT, _device="/job:localhost/replica:0/task:0/cpu:0"](prediction/Squeeze, prediction/ToFloat)]]

Caused by op 'prediction/logistic_loss/mul', defined at: File "udc_train.py", line 64, in tf.app.run() File "/usr/local/lib/python3.4/dist-packages/tensorflow/python/platform/app.py", line 43, in run sys.exit(main(sys.argv[:1] + flags_passthrough)) File "udc_train.py", line 61, in main estimator.fit(input_fn=input_fn_train, steps=None, monitors=[eval_monitor]) File "/usr/local/lib/python3.4/dist-packages/tensorflow/python/util/deprecation.py", line 191, in new_func return func(*args, **kwargs) File "/usr/local/lib/python3.4/dist-packages/tensorflow/contrib/learn/python/learn/estimators/estimator.py", line 355, in fit max_steps=max_steps) File "/usr/local/lib/python3.4/dist-packages/tensorflow/contrib/learn/python/learn/estimators/estimator.py", line 733, in _train_model max_steps=max_steps) File "/usr/local/lib/python3.4/dist-packages/tensorflow/contrib/learn/python/learn/graph_actions.py", line 301, in _monitored_train None) File "/usr/local/lib/python3.4/dist-packages/tensorflow/python/training/monitored_session.py", line 473, in run run_metadata=run_metadata) File "/usr/local/lib/python3.4/dist-packages/tensorflow/python/training/monitored_session.py", line 628, in run run_metadata=run_metadata) File "/usr/local/lib/python3.4/dist-packages/tensorflow/python/training/monitored_session.py", line 595, in run return self._sess.run(*args, **kwargs) File "/usr/local/lib/python3.4/dist-packages/tensorflow/python/training/monitored_session.py", line 737, in run run_metadata=run_metadata)) File "/usr/local/lib/python3.4/dist-packages/tensorflow/contrib/learn/python/learn/monitors.py", line 1210, in after_run induce_stop = m.step_end(self._last_step, result) File "/usr/local/lib/python3.4/dist-packages/tensorflow/contrib/learn/python/learn/monitors.py", line 409, in step_end return self.every_n_step_end(step, output) File "/usr/local/lib/python3.4/dist-packages/tensorflow/contrib/learn/python/learn/monitors.py", line 710, in every_n_step_end steps=self.eval_steps, metrics=self.metrics, name=self.name) File "/usr/local/lib/python3.4/dist-packages/tensorflow/python/util/deprecation.py", line 191, in new_func return func(*args, **kwargs) File "/usr/local/lib/python3.4/dist-packages/tensorflow/contrib/learn/python/learn/estimators/estimator.py", line 427, in evaluate name=name) File "/usr/local/lib/python3.4/dist-packages/tensorflow/contrib/learn/python/learn/estimators/estimator.py", line 793, in _evaluate_model eval_ops = self._get_eval_ops(features, labels, metrics) File "/usr/local/lib/python3.4/dist-packages/tensorflow/contrib/learn/python/learn/estimators/estimator.py", line 1080, in _get_eval_ops features, labels, model_fn_lib.ModeKeys.EVAL) File "/usr/local/lib/python3.4/dist-packages/tensorflow/contrib/learn/python/learn/estimators/estimator.py", line 1021, in _call_model_fn model_fn_results = self._model_fn(features, labels, mode=mode) File "/usr/src/chatbot-retrieval/udc_model.py", line 83, in model_fn tf.concat(0, all_targets)) File "/usr/src/chatbot-retrieval/models/dual_encoder.py", line 81, in dual_encoder_model losses = tf.nn.sigmoid_cross_entropy_with_logits(logits, tf.to_float(targets)) File "/usr/local/lib/python3.4/dist-packages/tensorflow/python/ops/nn.py", line 456, in sigmoid_cross_entropy_with_logits return math_ops.add(relu_logits - logits * targets, File "/usr/local/lib/python3.4/dist-packages/tensorflow/python/ops/math_ops.py", line 814, in binary_op_wrapper return func(x, y, name=name) File "/usr/local/lib/python3.4/dist-packages/tensorflow/python/ops/math_ops.py", line 987, in _mul_dispatch return gen_math_ops.mul(x, y, name=name) File "/usr/local/lib/python3.4/dist-packages/tensorflow/python/ops/gen_math_ops.py", line 1613, in mul result = _op_def_lib.apply_op("Mul", x=x, y=y, name=name) File "/usr/local/lib/python3.4/dist-packages/tensorflow/python/framework/op_def_library.py", line 759, in apply_op op_def=op_def) File "/usr/local/lib/python3.4/dist-packages/tensorflow/python/framework/ops.py", line 2240, in create_op original_op=self._default_original_op, op_def=op_def) File "/usr/local/lib/python3.4/dist-packages/tensorflow/python/framework/ops.py", line 1128, in init self._traceback = _extract_stack()

InvalidArgumentError (see above for traceback): Incompatible shapes: [80,1] vs. [160,1] [[Node: prediction/logistic_loss/mul = Mul[T=DT_FLOAT, _device="/job:localhost/replica:0/task:0/cpu:0"](prediction/Squeeze, prediction/ToFloat)]]

[rrivera1849]

Getting the same error, did you ever figure out what it was?

[vaibhavgeek]

I have the same error.

[vaibhavgeek]

Please check #15 issue.

[pengcao]

I have the same error. How to reslove? Thanks!