transformer training error

Traceback (most recent call last): File "/home/kinda/.local/lib/python3.6/site-packages/tensorflow/python/client/session.py", line 1334, in _do_call return fn(*args) File "/home/kinda/.local/lib/python3.6/site-packages/tensorflow/python/client/session.py", line 1319, in _run_fn options, feed_dict, fetch_list, target_list, run_metadata) File "/home/kinda/.local/lib/python3.6/site-packages/tensorflow/python/client/session.py", line 1407, in _call_tf_sessionrun run_metadata) tensorflow.python.framework.errors_impl.InvalidArgumentError: indices[28,100] = 100 is not in [0, 100) [[{{node encoder_1/positional_encoding/embedding_lookup}} = GatherV2[Taxis=DT_INT32, Tindices=DT_INT32, Tparams=DT_FLOAT, _device="/job:localhost/replica:0/task:0/device:CPU:0"](decoder_100/positional_encoding/Const, encoder_1/positional_encoding/Tile, encoder_1/positional_encoding/embedding_lookup/axis)]]

During handling of the above exception, another exception occurred:

Traceback (most recent call last): File "train.py", line 80, in hypotheses = get_hypotheses(num_eval_batches, num_eval_samples, sess, y_hat, m.idx2token) File "/home/kinda/deeplearning/transformer/utils.py", line 153, in get_hypotheses h = sess.run(tensor) File "/home/kinda/.local/lib/python3.6/site-packages/tensorflow/python/client/session.py", line 929, in run run_metadata_ptr) File "/home/kinda/.local/lib/python3.6/site-packages/tensorflow/python/client/session.py", line 1152, in _run feed_dict_tensor, options, run_metadata) File "/home/kinda/.local/lib/python3.6/site-packages/tensorflow/python/client/session.py", line 1328, in _do_run run_metadata) File "/home/kinda/.local/lib/python3.6/site-packages/tensorflow/python/client/session.py", line 1348, in _do_call raise type(e)(node_def, op, message) tensorflow.python.framework.errors_impl.InvalidArgumentError: indices[28,100] = 100 is not in [0, 100) [[node encoder_1/positional_encoding/embedding_lookup (defined at /home/kinda/deeplearning/transformer/modules.py:302) = GatherV2[Taxis=DT_INT32, Tindices=DT_INT32, Tparams=DT_FLOAT, _device="/job:localhost/replica:0/task:0/device:CPU:0"](decoder_100/positional_encoding/Const, encoder_1/positional_encoding/Tile, encoder_1/positional_encoding/embedding_lookup/axis)]]

Caused by op 'encoder_1/positional_encoding/embedding_lookup', defined at: File "train.py", line 47, in y_hat, eval_summaries = m.eval(xs, ys) File "/home/kinda/deeplearning/transformer/model.py", line 165, in eval memory, sents1 = self.encode(xs, False) File "/home/kinda/deeplearning/transformer/model.py", line 52, in encode enc += positional_encoding(enc, self.hp.maxlen1) File "/home/kinda/deeplearning/transformer/modules.py", line 302, in positional_encoding outputs = tf.nn.embedding_lookup(position_enc, position_ind) File "/home/kinda/.local/lib/python3.6/site-packages/tensorflow/python/ops/embedding_ops.py", line 313, in embedding_lookup transform_fn=None) File "/home/kinda/.local/lib/python3.6/site-packages/tensorflow/python/ops/embedding_ops.py", line 133, in _embedding_lookup_and_transform result = _clip(array_ops.gather(params[0], ids, name=name), File "/home/kinda/.local/lib/python3.6/site-packages/tensorflow/python/ops/array_ops.py", line 2675, in gather return gen_array_ops.gather_v2(params, indices, axis, name=name) File "/home/kinda/.local/lib/python3.6/site-packages/tensorflow/python/ops/gen_array_ops.py", line 3332, in gather_v2 "GatherV2", params=params, indices=indices, axis=axis, name=name) File "/home/kinda/.local/lib/python3.6/site-packages/tensorflow/python/framework/op_def_library.py", line 787, in _apply_op_helper op_def=op_def) File "/home/kinda/.local/lib/python3.6/site-packages/tensorflow/python/util/deprecation.py", line 488, in new_func return func(*args, **kwargs) File "/home/kinda/.local/lib/python3.6/site-packages/tensorflow/python/framework/ops.py", line 3274, in create_op op_def=op_def) File "/home/kinda/.local/lib/python3.6/site-packages/tensorflow/python/framework/ops.py", line 1770, in init self._traceback = tf_stack.extract_stack()

InvalidArgumentError (see above for traceback): indices[28,100] = 100 is not in [0, 100) [[node encoder_1/positional_encoding/embedding_lookup (defined at /home/kinda/deeplearning/transformer/modules.py:302) = GatherV2[Taxis=DT_INT32, Tindices=DT_INT32, Tparams=DT_FLOAT, _device="/job:localhost/replica:0/task:0/device:CPU:0"](decoder_100/positional_encoding/Const, encoder_1/positional_encoding/Tile, encoder_1/positional_encoding/embedding_lookup/axis)]]