the result of eval is poor

[Enlistedman]

Hello,I tested the model on ILSVRC2015 using the models/lstm5/WM-1.0-test.pth you provided, but each time the scores and boxes are the same value, the final result is very poor, can you provide the corresponding help? thank you very much.

[vikrant7]

Hi @Enlistedman , The model in models/lstm5/WM-1.0-test.pth is just an untrained model saved to verify my code. Sorry for the confusion, I will remove this files. Also please wait for few days as training is still going on, I will update README once it is done.

[Enlistedman]

Hi @Enlistedman , The model in models/lstm5/WM-1.0-test.pth is just an untrained model saved to verify my code. Sorry for the confusion, I will remove this files. Also please wait for few days as training is still going on, I will update README once it is done.

Thank you for your reply, I am looking forward to your training results.Best wishes for you.

[vikrant7]

Hi @Enlistedman , Can you please share the evaluation script used by you?

[Enlistedman]

@Enlistedman #!/usr/bin/python3 """Script for evaluation of trained model on Imagenet VID 2015 dataset. Few global variables defined here are explained: Global Variables

args : dict Has all the options for changing various variables of the model as well as parameters for evaluation dataset : ImagenetDataset (torch.utils.data.Dataset, For more info see datasets/vid_dataset.py)

""" import torch from network import mvod_basenet, mvod_bottleneck_lstm1, mvod_bottleneck_lstm2, mvod_bottleneck_lstm3, mvod_lstm4, mvod_lstm5 from network.predictor import Predictor from datasets.vid_dataset import ImagenetDataset,VIDDataset from config import mobilenetv1_ssd_config from utils import box_utils, measurements from utils.misc import str2bool, Timer import argparse import pathlib import numpy as np import logging import sys import cv2

parser = argparse.ArgumentParser(description="MVOD Evaluation on VID dataset") parser.add_argument('--net', default="lstm5", help="The network architecture, it should be of basenet, lstm1, lstm2, lstm3, lstm4 or lstm5.") parser.add_argument("--trained_model", type=str) parser.add_argument("--dataset", type=str, help="The root directory of the VOC dataset or Open Images dataset.") parser.add_argument("--label_file", type=str, help="The label file path.") parser.add_argument("--use_cuda", type=str2bool, default=True) parser.add_argument("--nms_method", type=str, default="hard") parser.add_argument("--iou_threshold", type=float, default=0.5, help="The threshold of Intersection over Union.") parser.add_argument("--eval_dir", default="eval_results_lstm5", type=str, help="The directory to store evaluation results.") parser.add_argument('--width_mult', default=1.0, type=float, help='Width Multiplifier for network') args = parser.parse_args() DEVICE = torch.device("cuda:0" if torch.cuda.is_available() and args.use_cuda else "cpu")

def group_annotation_by_class(dataset): """ Groups annotations of dataset by class """ true_case_stat = {} all_gt_boxes = {} all_difficult_cases = {} for i in range(len(dataset)): image_id, annotation = dataset.get_annotation(i) #gt_boxes, classes, is_difficult = annotation gt_boxes, classes = annotation is_difficult = [0] gt_boxes = torch.from_numpy(gt_boxes) for i, difficult in enumerate(is_difficult): class_index = int(classes[i]) gt_box = gt_boxes[i] if not difficult: true_case_stat[class_index] = true_case_stat.get(class_index, 0) + 1

		if class_index not in all_gt_boxes:
			all_gt_boxes[class_index] = {}
		if image_id not in all_gt_boxes[class_index]:
			all_gt_boxes[class_index][image_id] = []
		all_gt_boxes[class_index][image_id].append(gt_box)
		if class_index not in all_difficult_cases:
			all_difficult_cases[class_index]={}
		if image_id not in all_difficult_cases[class_index]:
			all_difficult_cases[class_index][image_id] = []
		all_difficult_cases[class_index][image_id].append(difficult)

for class_index in all_gt_boxes:
	for image_id in all_gt_boxes[class_index]:
		all_gt_boxes[class_index][image_id] = torch.stack(all_gt_boxes[class_index][image_id])
for class_index in all_difficult_cases:
	for image_id in all_difficult_cases[class_index]:
		all_gt_boxes[class_index][image_id] = torch.tensor(all_gt_boxes[class_index][image_id])
return true_case_stat, all_gt_boxes, all_difficult_cases

def compute_average_precision_per_class(num_true_cases, gt_boxes, difficult_cases, prediction_file, iou_threshold, use_2007_metric): """ Computes average precision per class """ with open(prediction_file) as f: image_ids = [] boxes = [] scores = [] for line in f: t = line.rstrip().split(" ") image_ids.append(t[0]) scores.append(float(t[1])) box = torch.tensor([float(v) for v in t[2:]]).unsqueeze(0) box -= 1.0 # convert to python format where indexes start from 0 boxes.append(box) scores = np.array(scores) sorted_indexes = np.argsort(-scores) boxes = [boxes[i] for i in sorted_indexes] image_ids = [image_ids[i] for i in sorted_indexes] true_positive = np.zeros(len(image_ids)) false_positive = np.zeros(len(image_ids)) matched = set() for i, image_id in enumerate(image_ids): box = boxes[i] if image_id not in gt_boxes: false_positive[i] = 1 continue

		gt_box = gt_boxes[image_id]
		ious = box_utils.iou_of(box, gt_box)
		max_iou = torch.max(ious).item()
		max_arg = torch.argmax(ious).item()
		if max_iou > iou_threshold:
			if difficult_cases[image_id][max_arg] == 0:
				if (image_id, max_arg) not in matched:
					true_positive[i] = 1
					matched.add((image_id, max_arg))
				else:
					false_positive[i] = 1
		else:
			false_positive[i] = 1

true_positive = true_positive.cumsum()
false_positive = false_positive.cumsum()
precision = true_positive / (true_positive + false_positive)
recall = true_positive / num_true_cases
if use_2007_metric:
	return measurements.compute_voc2007_average_precision(precision, recall)
else:
	return measurements.compute_average_precision(precision, recall)

if name == 'main': eval_path = pathlib.Path(args.eval_dir) eval_path.mkdir(exist_ok=True) timer = Timer() class_names = [name.strip() for name in open(args.label_file).readlines()] num_classes = len(class_names) dataset = ImagenetDataset(args.dataset, is_val=True) config = mobilenetv1_ssd_config true_case_stat, all_gb_boxes, all_difficult_cases = group_annotation_by_class(dataset) if args.net == 'basenet': pred_enc = mvod_basenet.MobileNetV1(num_classes=num_classes, alpha = args.width_mult) pred_dec = mvod_basenet.SSD(num_classes=num_classes, alpha = args.width_mult, is_test=True, config= config, batch_size=1) net = mvod_basenet.MobileVOD(pred_enc, pred_dec) elif args.net == 'lstm1': pred_enc = mvod_bottleneck_lstm1.MobileNetV1(num_classes=num_classes, alpha = args.width_mult) pred_dec = mvod_bottleneck_lstm1.SSD(num_classes=num_classes, alpha = args.width_mult, is_test=True, config= config, batch_size=1) net = mvod_bottleneck_lstm1.MobileVOD(pred_enc, pred_dec) elif args.net == 'lstm2': pred_enc = mvod_bottlneck_lstm2.MobileNetV1(num_classes=num_classes, alpha = args.width_mult) pred_dec = mvod_bottlneck_lstm2.SSD(num_classes=num_classes, alpha = args.width_mult, is_test=True, config= config, batch_size=1) net = mvod_bottlneck_lstm2.MobileVOD(pred_enc, pred_dec) elif args.net == 'lstm3': pred_enc = mvod_bottlneck_lstm3.MobileNetV1(num_classes=num_classes, alpha = args.width_mult) pred_dec = mvod_bottlneck_lstm3.SSD(num_classes=num_classes, alpha = args.width_mult, is_test=True, config= config, batch_size=1) net = mvod_bottlneck_lstm3.MobileVOD(pred_enc, pred_dec) elif args.net == 'lstm4': pred_enc = mvod_lstm4.MobileNetV1(num_classes=num_classes, alpha = args.width_mult) pred_dec = mvod_lstm4.SSD(num_classes=num_classes, alpha = args.width_mult, is_test=True, config= config, batch_size=1) net = mvod_lstm4.MobileVOD(pred_enc, pred_dec) elif args.net == 'lstm5': pred_enc = mvod_lstm5.MobileNetV1(num_classes=num_classes, alpha = args.width_mult) pred_dec = mvod_lstm5.SSD(num_classes=num_classes, alpha = args.width_mult, is_test=True, config= config, batch_size=1) net = mvod_lstm5.MobileVOD(pred_enc, pred_dec) else: logging.fatal("The net type is wrong. It should be one of basenet, lstm{1,2,3,4,5}.") parser.print_help(sys.stderr) sys.exit(1)

timer.start("Load Model")
net.load_state_dict(
		torch.load(args.trained_model,
				   map_location=lambda storage, loc: storage))
net = net.to(DEVICE)
print(f"It took {timer.end('Load Model')} seconds to load the model.")
predictor = Predictor(net, config.image_size, config.image_mean,
                      config.image_std,
                      nms_method=args.nms_method,
                      iou_threshold=config.iou_threshold,
                      candidate_size=200,
                      sigma=0.5,
                      device=DEVICE)

results = []
for i in range(len(dataset)):
	print("process image", i)
	timer.start("Load Image")
	image = dataset.get_image(i)
	
	print("Load Image: {:4f} seconds.".format(timer.end("Load Image")))
	timer.start("Predict")
	boxes, labels, probs = predictor.predict(image)
	if args.net == 'basenet':
		continue
	else:
		net.detach_hidden()
	print("Prediction: {:4f} seconds.".format(timer.end("Predict")))
	indexes = torch.ones(labels.size(0), 1, dtype=torch.float32) * i
	results.append(torch.cat([
		indexes.reshape(-1, 1),
		labels.reshape(-1, 1).float(),
		probs.reshape(-1, 1),
		boxes + 1.0  # matlab's indexes start from 1
	], dim=1))
results = torch.cat(results)
for class_index, class_name in enumerate(class_names):
	if class_index == 0: continue  # ignore background
	prediction_path = eval_path / f"det_test_{class_name}.txt"
	with open(prediction_path, "w") as f:
		sub = results[results[:, 1] == class_index, :]
		for i in range(sub.size(0)):
			print(sub[i,:].numpy())
			prob_box = sub[i, 2:].numpy()
			image_id = dataset.ids[int(sub[i, 0])]
			print(
				image_id + " " + " ".join([str(v) for v in prob_box]),
				file=f
			)
aps = []
print("\n\nAverage Precision Per-class:")
for class_index, class_name in enumerate(class_names):
	if class_index == 0:
		continue
	prediction_path = eval_path / f"det_test_{class_name}.txt"
	ap = compute_average_precision_per_class(
		true_case_stat[class_index],
		all_gb_boxes[class_index],
		all_difficult_cases[class_index],
		prediction_path,
		args.iou_threshold,
		use_2007_metric=False
	)
	aps.append(ap)
	print(f"{class_name}: {ap}")

print(f"\nAverage Precision Across All Classes:{sum(aps)/len(aps)}")

[Enlistedman]

@vikrant7 Hello,is all the code done?