[FR] Add App sliders for parameters other than confidence for detections

[qwertyman30]

Hi, I have parameters like depth, occlusion etc in my pedestrian detection data and I am trying to add sliders for these parameters in my data to each detection box. Since there are multiple detections per image, I cant add them as other fields to achieve the slider functionality as that seems to work per sample instead of per detection. Thus I am trying to extend the codebase to get this to work for my usecase. So far, i have modified the labels.detection class, some functions in fiftyone.utils.eta and eta.core.objects to include the new parameters as non attributes. I have also added the slider code in LabelFieldFilter.tsx file and added the path for the new parameters. I am importing the data using FiftyOneImageDetectionDatasetImporter. So I also changed ImageDetectionSampleParser to account for the detection changes.

Changes in labels.Detection

class Detection(_HasID, _HasAttributesDict, Label):
    meta = {"allow_inheritance": True}
    label = fof.StringField()
    bounding_box = fof.ListField(fof.FloatField())
    mask = fof.ArrayField()
    confidence = fof.FloatField()
    depth = fof.FloatField()
    index = fof.IntField()

Changes in fiftyone.utils.eta

def to_detected_object(detection, name=None):
      label = detection.label
      index = detection.index
      tlx, tly, w, h = detection.bounding_box
      brx = tlx + w
      bry = tly + h
      bounding_box = etag.BoundingBox.from_coords(tlx, tly, brx, bry) 
      mask = detection.mask
      confidence = detection.confidence
      depth = detection.depth  
      attrs = _to_eta_attributes(detection)
      return etao.DetectedObject(
          label=label,
          index=index,
          bounding_box=bounding_box,
          mask=mask,
          confidence=confidence,
          depth=depth,
          name=name,
          attrs=attrs,
          tags=detection.tags,
      )

  def from_detected_object(dobj):
      xtl, ytl, xbr, ybr = dobj.bounding_box.to_coords()
      bounding_box = [xtl, ytl, (xbr - xtl), (ybr - ytl)]
      attributes = _from_eta_attributes(dobj.attrs)
      return fol.Detection(
          label=dobj.label,
          bounding_box=bounding_box,
          confidence=dobj.confidence,
          depth=dobj.depth,
          index=dobj.index,
          mask=dobj.mask,
          tags=dobj.tags,
          **attributes,
      )

Changes in eta.core.objects

class DetectedObject(etal.Labels, etag.HasBoundingBox):
    def __init__(
            self,
            label=None,
            bounding_box=None,
            mask=None,
            confidence=None,
            depth=None,
            name=None,
            top_k_probs=None,
            index=None,
            score=None,
            frame_number=None,
            index_in_frame=None,
            eval_type=None,
            attrs=None,
            tags=None,
    ):
        self.type = etau.get_class_name(self)
        self.label = label
        self.bounding_box = bounding_box
        self.mask = mask
        self.confidence = confidence
        self.depth = depth
        self.name = name
        self.top_k_probs = top_k_probs
        self.index = index
        self.score = score
        self.frame_number = frame_number
        self.index_in_frame = index_in_frame
        self.eval_type = eval_type
        self.attrs = attrs or etad.AttributeContainer()
        self.tags = tags or []
        self._meta = None  # Usable by clients to store temporary metadata

    @property
    def has_depth(self):
        """Whether the object has a ``depth``."""
        return self.depth is not None

    def attributes(self):
        _attrs = ["type"]
        _noneable_attrs = [
            "label",
            "bounding_box",
            "mask",
            "confidence",
            "depth",
            "name",
            "top_k_probs",
            "index",
            "score",
            "frame_number",
            "index_in_frame",
            "eval_type",
            "tags",
        ]
        _attrs.extend(
            [a for a in _noneable_attrs if getattr(self, a) is not None]
        )
        if self.attrs:
            _attrs.append("attrs")
        return _attrs

    @classmethod
    def _from_dict(cls, d):
        bounding_box = d.get("bounding_box", None)
        if bounding_box is not None:
            bounding_box = etag.BoundingBox.from_dict(bounding_box)

        mask = d.get("mask", None)
        if mask is not None:
            mask = etas.deserialize_numpy_array(mask)

        attrs = d.get("attrs", None)
        if attrs is not None:
            attrs = etad.AttributeContainer.from_dict(attrs)

        return cls(
            label=d.get("label", None),
            bounding_box=bounding_box,
            mask=mask,
            confidence=d.get("confidence", None),
            depth=d.get("depth", None),
            name=d.get("name", None),
            top_k_probs=d.get("top_k_probs", None),
            index=d.get("index", None),
            score=d.get("score", None),
            frame_number=d.get("frame_number", None),
            index_in_frame=d.get("index_in_frame", None),
            attrs=attrs,
            eval_type=d.get("eval_type", None),
            tags=d.get("tags", None),
        )

Changes in parser

class ImageDetectionSampleParser(LabeledImageTupleSampleParser):
    def __init__(
            self,
            label_field="label",
            bounding_box_field="bounding_box",
            confidence_field=None,
            depth_field=None,
            attributes_field=None,
            classes=None,
            normalized=True,
    ):
        super().__init__()
        self.label_field = label_field
        self.bounding_box_field = bounding_box_field
        self.confidence_field = confidence_field
        self.depth_field = depth_field
        self.attributes_field = attributes_field
        self.classes = classes
        self.normalized = normalized

    def _parse_detection(self, obj, img=None):
        label = obj[self.label_field]

        try:
            label = self.classes[label]
        except:
            label = str(label)

        tlx, tly, w, h = self._parse_bbox(obj)

        if not self.normalized:
            height, width = img.shape[:2]
            tlx /= width
            tly /= height
            w /= width
            h /= height

        bounding_box = [tlx, tly, w, h]

        if self.confidence_field:
            confidence = obj.get(self.confidence_field, None)
        else:
            confidence = None

        if self.depth_field:
            depth = obj.get(self.depth_field, None)
        else:
            depth = None

        if self.attributes_field:
            attributes = obj.get(self.attributes_field, {})
        else:
            attributes = {}

        return fol.Detection(
            label=label,
            bounding_box=bounding_box,
            confidence=confidence,
            depth=depth,
            **attributes,
        )

These were the major changes in python codebase. I have duplicated the sliders code in typescript just below the confidence slider in LabelFieldSlider.tsx and changed the path for the slider to ${path}.depth. Also added the depth code wherever confidence was referenced in the app. As seen in the attached image, for some reason, depth still shows up only as an attribute that can be viewed when hovered over the detection and the app is not able to add the depth slider per detection in the labels. So I wanted to know if per detection sliders for params other than confidence is possible already. If not can somebody point me to the necessary changes i need to make to add custom sliders.

P.S. I will create custom classes and functions again or make this more general i.e. creating sliders for certain types of fields (eg. float and int fields only) before creating a pull request. I just wanted to get the code working for my usecase first.

[benjaminpkane]

@qwertyman30 I am working on this via dynamic schema expansion in labels. I think you are on the right track, but we want to support filter widgets more generally on labels. So my changes will include an overhaul in the App that creates label filter widgets with respect to the dynamic schema.

These changes will support your use case.

[brimoor]

@qwertyman30 wow this is amazing! It's awesome that you dove in so deep to address your needs 🔥 💯 🥇

Like Ben mentioned, it just so happens that we were already working on supporting exactly what you wanted here! Within the next few days you'll automatically get filters for any custom fields that you add to your Label instances.

Also, note that FiftyOneImageDetectionDataset already supports loading custom attributes. You just need to store them in the attributes field. This snippet demonstrates:

import random

import fiftyone as fo
import fiftyone.zoo as foz

dataset = fo.Dataset()

dataset = (
    foz.load_zoo_dataset("quickstart", max_samples=1, shuffle=True)
    .select_fields("ground_truth")
).clone()

sample = dataset.first()
for detection in sample.ground_truth.detections:
    detection.depth = random.random()
    detection.occluded = random.random() < 0.5
    detection.sex = random.choice(["male", "female"])

sample.save()

dataset.export(
    export_dir="/tmp/test",
    dataset_type=fo.types.FiftyOneImageDetectionDataset,
)

dataset2 = fo.Dataset.from_dir(
    dataset_dir="/tmp/test",
    dataset_type=fo.types.FiftyOneImageDetectionDataset,
)

The /tmp/test/labels.json file that was generated and then imported back looks like this:

{
    "classes": null,
    "labels": {
        "000083": [{
            "label": "knife",
            "bounding_box": [0.001265625, 0.4381894150417827, 0.22940624999999998, 0.2786629526462396],
            "attributes": {
                "iscrowd": 0.0,
                "occluded": false,
                "area": 2875.5717500000014,
                "sex": "male",
                "depth": 0.17504318631735394
            }
        }, {
            "label": "carrot",
            "bounding_box": [0.266609375, 0.1952924791086351, 0.48439062499999996, 0.7505849582172701],
            "attributes": {
                "iscrowd": 0.0,
                "occluded": true,
                "area": 52420.690299999995,
                "sex": "male",
                "depth": 0.4788877755782093
            }
        }, {
            "label": "knife",
            "bounding_box": [0.010203125, 0.32208913649025067, 0.225828125, 0.12206128133704736],
            "attributes": {
                "iscrowd": 0.0,
                "occluded": true,
                "area": 2016.1827499999993,
                "sex": "female",
                "depth": 0.11556803149185702
            }
        }]
    }
}

[jasm37]

Hi @brimoor and @benjaminpkane. I see this is still open. I also have detection attributes besides confidence that do not have sliders in the App. Has this been somehow fixed or is there an update or workaround to make this work? My setup is Python 3.8.9, FiftyOne v0.16.5.

[brimoor]

Hey @jasm37 we are very close to adding support for this. https://github.com/voxel51/fiftyone/pull/1825 and https://github.com/voxel51/fiftyone/pull/1826 are two alternatives we are considering for adding dynamic attributes to a dataset's schema. Once that is supported, those attributes will be filterable from the App.

I would like to get this into the next release or two.

[brimoor]

In the meantime, the workaround is to construct the relevant view via Python and load it in session.view like so:

import fiftyone as fo
from fiftyone import ViewField as F

dataset = fo.load_dataset(...)
session = fo.launch_app(dataset)

# Slider
session.view = dataset.filter_labels(
    "ground_truth",
    (F("custom") >= 0.5) & (F("custom") <= 0.9),
)

# Selector
session.view = dataset.filter_labels(
    "ground_truth",
    F("custom").is_in(["list", "of", "values"]),
)