microsoft
Correction to IOU calcuation
Findings
Found that grits.iou function is outputting values that are slightly off compared to other libraries. Updated function to correct output. Additionally a duplicate iou function was found in the postprocessing.py file, so it was instead replaced with the function imported from the grits module instead.
Changes
Attempted to make minimal changes to fix the issue. Only the change in the function was tested, importing the function for reuse was not verified, but is expected to work.
Reproducing Errors
The following was used to verify that values calculated by original grits.iou funciton were incorrect. Over a few iterations of running the example, the maximum difference observed was around $0.089$ in the IoU score.
Expand for code used to check for errors
import torch
import numpy as np
from fitz import Rect
from torchvision.ops import box_iou
# Original function from grits.py module
def iou_tatr(bbox1: list, bbox2: list) -> float:
"""
Compute the intersection-over-union of two bounding boxes.
"""
intersection = Rect(bbox1).intersect(bbox2)
union = Rect(bbox1).include_rect(bbox2)
union_area = union.get_area()
if union_area > 0:
return intersection.get_area() / union.get_area()
return 0
# Function with changes made in this PR
def new_iou(bbox1: list, bbox2: list) -> float:
"""
Compute the intersection-over-union of two bounding boxes.
"""
intersection = Rect(bbox1).intersect(bbox2)
union_area = Rect(bbox1).get_area() + Rect(bbox2).get_area() - intersection.get_area()
if union_area > 0:
return intersection.get_area() / union_area
return 0
# Wrapper around torchvision.ops.box_iou function for comparison
def t_iou(bbox1 :list, bbox2: list) -> float:
"""
Compute the intersection-over-union of two bounding boxes using torchvision function.
"""
bbox1 = torch.tensor(bbox1)
bbox2 = torch.tensor(bbox2)
bbox1 = bbox1.view(1, 4) if bbox1.dim() == 1 else bbox1
bbox2 = bbox2.view(1, 4) if bbox2.dim() == 1 else bbox2
return box_iou(bbox1, bbox2).item()
# Create random bounding boxes
N = 10_000
boxes1 = np.array([[np.random.randint(0, 1000) for _ in range(4)] for _ in range(N)])
boxes2 = np.array(
[[np.clip(v + np.random.randint(-v // 4, (v // 4) + 1), 0, 1000) for v in b] for b in boxes1]
)
# Make sure that the boxes are valid
for i in range(2):
# Find values where boxes are invalid
neg1 = np.where(np.diff(boxes1[:, i::2]) < 0)[0]
neg2 = np.where(np.diff(boxes2[:, i::2]) < 0)[0]
# Swap values to ensure boxes are valid
boxes1[neg1, i:i+1], boxes1[neg1, i+2:i+3] = boxes1[neg1, i+2:i+3], boxes1[neg1, i:i+1]
boxes2[neg2, i:i+1], boxes2[neg2, i+2:i+3] = boxes2[neg2, i+2:i+3], boxes2[neg2, i:i+1]
assert not ((np.diff(boxes1[:, 0::2]) < 0).any() or (np.diff(boxes2[:, 0::2] < 0).any())), "Negative width"
assert not ((np.diff(boxes1[:, 1::2]) < 0).any() or (np.diff(boxes2[:, 1::2] < 0).any())), "Negative height"
# Run calculations
results = {}
diffs = {}
for bi, (b1, b2) in enumerate(zip(boxes1, boxes2)):
tatr = iou_tatr(b1.tolist(), b2.tolist())
torchvis = t_iou(b1.tolist(), b2.tolist())
iou_2 = new_iou(b1.tolist(), b2.tolist())
results[bi] = {
"TATR-IOU": round(float(tatr), 5),
"TATR-IOU2": round(float(iou_2), 5),
"Torchvision-IOU": round(float(torchvis), 5),
}
diffs[bi] = (
round(float(abs(tatr - iou_2)), 5), # TATR - new_iou
round(float(abs(tatr - torchvis)), 5), # TATR - Torchvision
)
max([max(v) for v in diffs.values()])
# Generally max difference in IOU is ~0.089
Hopefully this is helpful for others. It's likely a minute difference in the scoring, but seemed like it could have an impact on model performance since iou scoring is used for both $GriTS_{top}$ and $GriTS_{loc}$.
