Error reporting using VOC dataset:RuntimeError: CUDA error: device-side assert triggered
Traceback (most recent call last):
File "/mnt/PyCharm_Project_2/tools/train_net.py", line 119, in
args=(args,),
File "/mnt/detectron2-0.4/detectron2/engine/launch.py", line 62, in launch
main_func(*args)
File "/mnt/PyCharm_Project_2/tools/train_net.py", line 107, in main
return trainer.train()
File "/mnt/PyCharm_Project_2/fsdet/engine/defaults.py", line 444, in train
super().train(self.start_iter, self.max_iter)
File "/mnt/detectron2-0.4/detectron2/engine/train_loop.py", line 138, in train
self.run_step()
File "/mnt/detectron2-0.4/detectron2/engine/train_loop.py", line 232, in run_step
loss_dict = self.model(data)
File "/root/miniconda3/envs/myconda/lib/python3.7/site-packages/torch/nn/modules/module.py", line 889, in _call_impl
result = self.forward(*input, **kwargs)
File "/mnt/PyCharm_Project_2/fsdet/modeling/meta_arch/rcnn.py", line 115, in forward
images, features, gt_instances
File "/root/miniconda3/envs/myconda/lib/python3.7/site-packages/torch/nn/modules/module.py", line 889, in _call_impl
result = self.forward(*input, **kwargs)
File "/mnt/detectron2-0.4/detectron2/modeling/proposal_generator/rpn.py", line 430, in forward
gt_labels, gt_boxes = self.label_and_sample_anchors(anchors, gt_instances)
File "/root/miniconda3/envs/myconda/lib/python3.7/site-packages/torch/autograd/grad_mode.py", line 27, in decorate_context
return func(*args, **kwargs)
File "/mnt/detectron2-0.4/detectron2/modeling/proposal_generator/rpn.py", line 301, in label_and_sample_anchors
matched_idxs, gt_labels_i = retry_if_cuda_oom(self.anchor_matcher)(match_quality_matrix)
File "/mnt/detectron2-0.4/detectron2/utils/memory.py", line 70, in wrapped
return func(*args, **kwargs)
File "/mnt/detectron2-0.4/detectron2/modeling/matcher.py", line 87, in call
assert torch.all(match_quality_matrix >= 0)
I have the same question。Did you solved it？
ucbdrive
