Error while evaluating saved model

[stefat77]

Hi, I'm trying to restore a model saved with tfdeploy but during evaluation I have the following errors:

<class 'tfdeploy.Tensor'>
<class 'tfdeploy.Tensor'>
Traceback (most recent call last):
  File "ensemble_learning.py", line 71, in <module>
    result = y_test.eval({x_test: test_waves})
  File "/home/stefano/Dropbox/DeepWave/tfdeploy.py", line 292, in eval
    self.value = self.op.eval(feed_dict=feed_dict, _uuid=_uuid)[self.value_index]
  File "/home/stefano/Dropbox/DeepWave/tfdeploy.py", line 461, in eval
    args = [t.eval(feed_dict=feed_dict, _uuid=_uuid) for t in self.inputs]
  File "/home/stefano/Dropbox/DeepWave/tfdeploy.py", line 461, in <listcomp>
    args = [t.eval(feed_dict=feed_dict, _uuid=_uuid) for t in self.inputs]
  File "/home/stefano/Dropbox/DeepWave/tfdeploy.py", line 292, in eval
    self.value = self.op.eval(feed_dict=feed_dict, _uuid=_uuid)[self.value_index]
  File "/home/stefano/Dropbox/DeepWave/tfdeploy.py", line 461, in eval
    args = [t.eval(feed_dict=feed_dict, _uuid=_uuid) for t in self.inputs]
  File "/home/stefano/Dropbox/DeepWave/tfdeploy.py", line 461, in <listcomp>
    args = [t.eval(feed_dict=feed_dict, _uuid=_uuid) for t in self.inputs]
  File "/home/stefano/Dropbox/DeepWave/tfdeploy.py", line 292, in eval
    self.value = self.op.eval(feed_dict=feed_dict, _uuid=_uuid)[self.value_index]
  File "/home/stefano/Dropbox/DeepWave/tfdeploy.py", line 461, in eval
    args = [t.eval(feed_dict=feed_dict, _uuid=_uuid) for t in self.inputs]
  File "/home/stefano/Dropbox/DeepWave/tfdeploy.py", line 461, in <listcomp>
    args = [t.eval(feed_dict=feed_dict, _uuid=_uuid) for t in self.inputs]
  File "/home/stefano/Dropbox/DeepWave/tfdeploy.py", line 292, in eval
    self.value = self.op.eval(feed_dict=feed_dict, _uuid=_uuid)[self.value_index]
  File "/home/stefano/Dropbox/DeepWave/tfdeploy.py", line 461, in eval
    args = [t.eval(feed_dict=feed_dict, _uuid=_uuid) for t in self.inputs]
  File "/home/stefano/Dropbox/DeepWave/tfdeploy.py", line 461, in <listcomp>
    args = [t.eval(feed_dict=feed_dict, _uuid=_uuid) for t in self.inputs]
  File "/home/stefano/Dropbox/DeepWave/tfdeploy.py", line 292, in eval
    self.value = self.op.eval(feed_dict=feed_dict, _uuid=_uuid)[self.value_index]
  File "/home/stefano/Dropbox/DeepWave/tfdeploy.py", line 461, in eval
    args = [t.eval(feed_dict=feed_dict, _uuid=_uuid) for t in self.inputs]
  File "/home/stefano/Dropbox/DeepWave/tfdeploy.py", line 461, in <listcomp>
    args = [t.eval(feed_dict=feed_dict, _uuid=_uuid) for t in self.inputs]
  File "/home/stefano/Dropbox/DeepWave/tfdeploy.py", line 292, in eval
    self.value = self.op.eval(feed_dict=feed_dict, _uuid=_uuid)[self.value_index]
  File "/home/stefano/Dropbox/DeepWave/tfdeploy.py", line 461, in eval
    args = [t.eval(feed_dict=feed_dict, _uuid=_uuid) for t in self.inputs]
  File "/home/stefano/Dropbox/DeepWave/tfdeploy.py", line 461, in <listcomp>
    args = [t.eval(feed_dict=feed_dict, _uuid=_uuid) for t in self.inputs]
  File "/home/stefano/Dropbox/DeepWave/tfdeploy.py", line 292, in eval
    self.value = self.op.eval(feed_dict=feed_dict, _uuid=_uuid)[self.value_index]
  File "/home/stefano/Dropbox/DeepWave/tfdeploy.py", line 461, in eval
    args = [t.eval(feed_dict=feed_dict, _uuid=_uuid) for t in self.inputs]
  File "/home/stefano/Dropbox/DeepWave/tfdeploy.py", line 461, in <listcomp>
    args = [t.eval(feed_dict=feed_dict, _uuid=_uuid) for t in self.inputs]
  File "/home/stefano/Dropbox/DeepWave/tfdeploy.py", line 292, in eval
    self.value = self.op.eval(feed_dict=feed_dict, _uuid=_uuid)[self.value_index]
  File "/home/stefano/Dropbox/DeepWave/tfdeploy.py", line 461, in eval
    args = [t.eval(feed_dict=feed_dict, _uuid=_uuid) for t in self.inputs]
  File "/home/stefano/Dropbox/DeepWave/tfdeploy.py", line 461, in <listcomp>
    args = [t.eval(feed_dict=feed_dict, _uuid=_uuid) for t in self.inputs]
  File "/home/stefano/Dropbox/DeepWave/tfdeploy.py", line 292, in eval
    self.value = self.op.eval(feed_dict=feed_dict, _uuid=_uuid)[self.value_index]
  File "/home/stefano/Dropbox/DeepWave/tfdeploy.py", line 461, in eval
    args = [t.eval(feed_dict=feed_dict, _uuid=_uuid) for t in self.inputs]
  File "/home/stefano/Dropbox/DeepWave/tfdeploy.py", line 461, in <listcomp>
    args = [t.eval(feed_dict=feed_dict, _uuid=_uuid) for t in self.inputs]
  File "/home/stefano/Dropbox/DeepWave/tfdeploy.py", line 292, in eval
    self.value = self.op.eval(feed_dict=feed_dict, _uuid=_uuid)[self.value_index]
  File "/home/stefano/Dropbox/DeepWave/tfdeploy.py", line 461, in eval
    args = [t.eval(feed_dict=feed_dict, _uuid=_uuid) for t in self.inputs]
  File "/home/stefano/Dropbox/DeepWave/tfdeploy.py", line 461, in <listcomp>
    args = [t.eval(feed_dict=feed_dict, _uuid=_uuid) for t in self.inputs]
  File "/home/stefano/Dropbox/DeepWave/tfdeploy.py", line 292, in eval
    self.value = self.op.eval(feed_dict=feed_dict, _uuid=_uuid)[self.value_index]
  File "/home/stefano/Dropbox/DeepWave/tfdeploy.py", line 461, in eval
    args = [t.eval(feed_dict=feed_dict, _uuid=_uuid) for t in self.inputs]
  File "/home/stefano/Dropbox/DeepWave/tfdeploy.py", line 461, in <listcomp>
    args = [t.eval(feed_dict=feed_dict, _uuid=_uuid) for t in self.inputs]
  File "/home/stefano/Dropbox/DeepWave/tfdeploy.py", line 292, in eval
    self.value = self.op.eval(feed_dict=feed_dict, _uuid=_uuid)[self.value_index]
  File "/home/stefano/Dropbox/DeepWave/tfdeploy.py", line 469, in eval
    self.value = self.func(*args)
  File "/home/stefano/Dropbox/DeepWave/tfdeploy.py", line 481, in func
    return cls.func_numpy(*args)
  File "/home/stefano/Dropbox/DeepWave/tfdeploy.py", line 1296, in Div
    return np.divide(a, b),
TypeError: unsupported operand type(s) for /: 'float' and 'NoneType'

The training code is:

...
deepnn definition, dataset loading....
...

td.setup(tf)
start_time = time.time()
with tf.Session() as sess:
    x = tf.placeholder(tf.float32, [None, 160], name="input")
    y = tf.placeholder(tf.float32, [None, 2], name="y-input")
    keep_prob = tf.placeholder(tf.float32)
    # Build the graph for the deep net
    y_conv = deepnn(x)
    y_soft = tf.nn.softmax(y_conv, name="output")
    prediction_y_conv = tf.argmax(y_conv, 1) # Predicted labels
    prediction_y = tf.argmax(y, 1) # Original labels

    with tf.name_scope('cost'):
        cross_entropy = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(labels=y, logits=y_conv)) #cost

    with tf.name_scope('train'):
    	train_step = tf.train.AdamOptimizer(LEARNING_RATE).minimize(cross_entropy) #optimizer

    with tf.name_scope('accuracy'):
        with tf.name_scope('correct_prediction'):
    	    correct_prediction = tf.equal(prediction_y_conv, prediction_y)
        with tf.name_scope('accuracy'):
    	    accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))

    # create a summary for our cost and accuracy
    tf.summary.scalar("accuracy", accuracy)
    tf.summary.scalar("cost", cross_entropy)
    merged = tf.summary.merge_all()

    writer = tf.summary.FileWriter(LOG_DIR, sess.graph)

    sess.run(tf.global_variables_initializer())



    best_accuracy = 0
    for e in range(num_epochs):
        print('Epoch num: ', e + 1)
        print('Learning rate: ', LEARNING_RATE)
        for i in range(num_batches):
            if i % 50 == 0:
                train_accuracy = sess.run(accuracy, feed_dict={x: train_waves[i], y: train_labels[i], keep_prob: 1.0})
                print('step %d, %d [s], training accuracy %g' % (i, (time.time() - start_time), train_accuracy))
            summary, _ = sess.run([merged,train_step], feed_dict={x: train_waves[i], y: train_labels[i], keep_prob: 1.0})
            writer.add_summary(summary,i)

        current_accuracy = accuracy.eval(feed_dict={x: test_waves, y: test_labels, keep_prob: 1.0})
        print('test accuracy %g' % current_accuracy)
        if current_accuracy > best_accuracy:
            # create a tfdeploy model and save it to disk
            model = td.Model()
            model.add(y_soft, sess) # y and all its ops and related tensors are added recursively
            model.save(SESS_DIR + '/' + MODEL_NAME)
            best_accuracy = current_accuracy

        # Compute confusion matrix
        cm_labels = prediction_y.eval(feed_dict={y: test_labels})
        cm_predictions = prediction_y_conv.eval(feed_dict={x: test_waves, keep_prob: 1.0})
        print(sess.run(tf.contrib.metrics.confusion_matrix(cm_labels,cm_predictions)))
        # Decrease learning rate with num_epochs
        #LEARNING_RATE = LEARNING_RATE - (LEARNING_RATE/100)*10
        print("Execution time [s]: %d" % (time.time() - start_time))

And the import code is:

....
load dataset
....
model = td.Model(SESS_DIR+"/model1.pkl")
#model = td.Model("model.pkl")

# shorthand to x and y
x_test, y_test = model.get("input", "output")
print(type(x_test))
print(type(y_test))

# evaluate
result = y_test.eval({x_test: test_waves})

Have I done something wrong? :| Thank you for your work!

EDIT: I forgot the dropout placeholder! :) In the training file i modified keep_prob: keep_prob = tf.placeholder(tf.float32, name="keep_prob")

Now the import code is:

ciao = td.Model(SESS_DIR+"/model1.pkl")
x, y, keep_prob = ciao.get("input", "output", "keep_prob")

# Get a prediction
pred = y.eval({x: test_waves, keep_prob: 1.0})

But now I have this error:

/home/stefano/Dropbox/DeepWave/tfdeploy.py:2081: RuntimeWarning: overflow encountered in exp
  e = np.exp(a)
/home/stefano/Dropbox/DeepWave/tfdeploy.py:2082: RuntimeWarning: invalid value encountered in true_divide
  return np.divide(e, np.sum(e, axis=-1, keepdims=True)),

EDIT2: I don't know why.... but now it works!! :D I had a "normalized" dataset (values between 0 and 1). Removed the normalization the problem is gone. Now i have a dataset with values between 0 and 255.