Loss is constantly NaN

[dberma15]

Hi,

I'm trying to train my dataset for the Data Science Bowl 2018 competition and I'm having trouble. The loss is always NaN no matter what I try. I know that my data set is structured properly, as it looks like this:

{'image': {'checksum': '2ce2a6891c485df5f5e1175b48cb305f', 'pathname': 'stage1_train/7f38885521586fc6011bef1314a9fb2aa1e4935bd581b2991e1d963395eab770/images/7f38885521586fc6011bef1314a9fb2aa1e4935bd581b2991e1d963395eab770.png', 'shape': {'r': 1024, 'c': 1024, 'channels': 3}}, 'objects': [{'bounding_box': {'minimum': {'r': 12, 'c': 14}, 'maximum': {'r': 14, 'c': 17}}, 'category': 'cell'}, {'bounding_box': {'minimum': {'r': 169, 'c': 109}, 'maximum': {'r': 170, 'c': 110}},  'category': 'cell'},....

So the problem isn't the data. Once I load it and try to run it, the model compiles, but the loss is consistently NaN and I can't figure out why. Can someone help?

import keras

import keras_rcnn.datasets.shape
import keras_rcnn.models
import keras_rcnn.preprocessing
import pickle
import numpy as np

def main():
	trainingdata = pickle.load(open('xtrdata.pkl','rb'))
	
	msk=np.random.random(len(trainingdata))<.9
	training_dictionary=[ trn for trn, m in zip(trainingdata,msk) if m]
	test_dictionary=[ trn for trn, m in zip(trainingdata,msk) if not m]
	# training_dictionary, test_dictionary = keras_rcnn.datasets.shape.load_data()

	categories = {"cell":1}

	generator = keras_rcnn.preprocessing.ObjectDetectionGenerator()

	generator = generator.flow_from_dictionary(
		dictionary=training_dictionary,
		categories=categories,
		target_size=(256, 256)
	)

	validation_data = keras_rcnn.preprocessing.ObjectDetectionGenerator()

	validation_data = validation_data.flow_from_dictionary(
		dictionary=test_dictionary,
		categories=categories,
		target_size=(256, 256)
	)

	keras.backend.set_learning_phase(1)

	model = keras_rcnn.models.RCNN(
		categories=["cell"],
		dense_units=512,
		input_shape=(256, 256, 3)
	)

	optimizer = keras.optimizers.Adam()
	model.compile(optimizer)
	model.save("test_rcnn.h5")
	model.fit_generator(
		epochs=100, steps_per_epoch=4,
		generator=generator,
		validation_data=validation_data)


if __name__ == '__main__':
	main()