test_box_with_nms_limit.py

#!/usr/bin/env python3
# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved


import unittest

import torch
from detectron2.layers import cat
from detectron2.modeling.roi_heads.fast_rcnn import fast_rcnn_inference
from detectron2.structures import Boxes
from mobile_cv.common.misc.oss_utils import is_oss


class TestBoxWithNMSLimit(unittest.TestCase):
    @unittest.skipIf(is_oss(), "Caffe2 is not available for OSS")
    def test_caffe2_pytorch_eq(self):
        ims_per_batch = 8
        post_nms_topk = 100
        detections_per_im = 10
        num_class = 80
        score_thresh = 0.05
        nms_thresh = 0.5

        image_shapes = [torch.Size([800, 800])] * ims_per_batch
        batch_splits = [post_nms_topk] * ims_per_batch

        # NOTE: There're still some unsure minor implementation differences
        # (eg. ordering when equal score across classes) causing some seeds
        # don't pass the test.
        # Thus set a fixed seed to make sure this test passes consistantly.
        rng = torch.Generator()
        rng.manual_seed(42)
        boxes = []
        for n in batch_splits:
            box = 1000.0 * 0.5 * torch.rand(n, num_class, 4, generator=rng) + 0.001
            box[:, :, -2:] += box[:, :, :2]
            box = box.view(n, num_class * 4)
            boxes.append(box)
        scores = [torch.rand(n, num_class + 1, generator=rng) for n in batch_splits]

        ref_results, ref_kept_indices = fast_rcnn_inference(
            boxes,
            scores,
            image_shapes,
            score_thresh=score_thresh,
            nms_thresh=nms_thresh,
            topk_per_image=detections_per_im,
        )
        for result, kept_index, score in zip(ref_results, ref_kept_indices, scores):
            torch.testing.assert_allclose(
                score[kept_index, result.pred_classes],
                result.scores,
            )

        # clip is done in BBoxTransformOp
        c2_boxes = []
        for box, image_shape in zip(boxes, image_shapes):
            num_bbox_reg_classes = box.shape[1] // 4
            clipped_box = Boxes(box.reshape(-1, 4))
            clipped_box.clip(image_shape)
            clipped_box = clipped_box.tensor.view(-1, num_bbox_reg_classes * 4)
            c2_boxes.append(clipped_box)

        c2_boxes = cat(c2_boxes)
        c2_scores = cat(scores)
        c2_batch_splits = torch.Tensor(batch_splits)

        nms_outputs = torch.ops._caffe2.BoxWithNMSLimit(
            c2_scores,
            c2_boxes,
            c2_batch_splits,
            score_thresh=float(score_thresh),
            nms=float(nms_thresh),
            detections_per_im=int(detections_per_im),
            soft_nms_enabled=False,
            soft_nms_method="linear",
            soft_nms_sigma=0.5,
            soft_nms_min_score_thres=0.001,
            rotated=False,
            cls_agnostic_bbox_reg=False,
            input_boxes_include_bg_cls=False,
            output_classes_include_bg_cls=False,
            legacy_plus_one=False,
        )
        (
            roi_score_nms,
            roi_bbox_nms,
            roi_class_nms,
            roi_batch_splits_nms,
            roi_keeps_nms,
            roi_keeps_size_nms,
        ) = nms_outputs  # noqa

        roi_score_nms = roi_score_nms.split(roi_batch_splits_nms.int().tolist())
        roi_bbox_nms = roi_bbox_nms.split(roi_batch_splits_nms.int().tolist())
        roi_class_nms = roi_class_nms.split(roi_batch_splits_nms.int().tolist())
        roi_keeps_nms = roi_keeps_nms.split(roi_batch_splits_nms.int().tolist())

        for _score_nms, _class_nms, _keeps_nms, _score in zip(
            roi_score_nms, roi_class_nms, roi_keeps_nms, scores
        ):
            torch.testing.assert_allclose(
                _score[_keeps_nms.to(torch.int64), _class_nms.to(torch.int64)],
                _score_nms,
            )

        for ref, s, b, c in zip(
            ref_results, roi_score_nms, roi_bbox_nms, roi_class_nms
        ):
            s1, i1 = s.sort()
            s2, i2 = ref.scores.sort()
            torch.testing.assert_allclose(s1, s2)
            torch.testing.assert_allclose(b[i1], ref.pred_boxes.tensor[i2])
            torch.testing.assert_allclose(c.to(torch.int64)[i1], ref.pred_classes[i2])

        for ref, k in zip(ref_kept_indices, roi_keeps_nms):
            # NOTE: order might be different due to implementation
            ref_set = set(ref.tolist())
            k_set = set(k.tolist())
            self.assertEqual(ref_set, k_set)