eval_utils.py

"""
ODinW evaluation utilities.
"""
import os
import json
import tempfile
import numpy as np
from typing import List, Dict, Sequence
from collections import OrderedDict
from pycocotools.coco import COCO
from pycocotools.cocoeval import COCOeval


def xyxy2xywh(bbox: np.ndarray) -> list:
    """Convert bbox format from xyxy to xywh.
    
    Args:
        bbox: Bounding box in [x1, y1, x2, y2] format
    
    Returns:
        Bounding box in [x, y, w, h] format
    """
    _bbox = bbox.tolist()
    return [
        _bbox[0],
        _bbox[1],
        _bbox[2] - _bbox[0],
        _bbox[3] - _bbox[1],
    ]


def results2json(results: Sequence[dict], outfile_prefix: str, cat_ids: dict) -> dict:
    """Convert results to COCO JSON format.
    
    Args:
        results: List of prediction results
        outfile_prefix: Output file prefix
        cat_ids: Category ID mapping
    
    Returns:
        result_files: Dictionary of result file paths
    """
    bbox_json_results = []
    for idx, result in enumerate(results):
        image_id = result.get('img_id', idx)
        labels = result['labels']
        bboxes = result['bboxes']
        scores = result['scores']
        
        for i, label in enumerate(labels):
            data = dict()
            data['image_id'] = image_id
            data['bbox'] = xyxy2xywh(bboxes[i])
            data['score'] = float(scores[i])
            data['category_id'] = cat_ids[label]
            bbox_json_results.append(data)
    
    result_files = dict()
    result_files['bbox'] = f'{outfile_prefix}.bbox.json'
    with open(result_files['bbox'], 'w') as f:
        json.dump(bbox_json_results, f)
    
    return result_files


def compute_metrics(results: list, outfile_prefix: str = None, _coco_api: COCO = None) -> Dict[str, float]:
    """Compute mAP and other metrics using COCO API.
    
    Args:
        results: List of evaluation results, each element is a (gt, pred) tuple
        outfile_prefix: Output file prefix (optional)
        _coco_api: COCO API instance
    
    Returns:
        eval_results: Dictionary of evaluation metrics
    """
    proposal_nums = (100, 300, 1000)
    iou_thrs = np.linspace(
        .5, 0.95, int(np.round((0.95 - .5) / .05)) + 1, endpoint=True)
    
    # Separate ground truth and predictions
    if len(results) == 0:
        gts, preds = [], []
    else:
        gts, preds = zip(*results)
    
    tmp_dir = None
    if outfile_prefix is None:
        tmp_dir = tempfile.TemporaryDirectory()
        outfile_prefix = os.path.join(tmp_dir.name, 'results')
    
    cat_ids = _coco_api.getCatIds()
    img_ids = _coco_api.getImgIds()
    
    # Convert to COCO format and save
    result_files = results2json(preds, outfile_prefix, cat_ids)
    
    eval_results = OrderedDict()
    
    for metric in ["bbox"]:
        iou_type = metric
        if metric not in result_files:
            raise KeyError(f'{metric} is not in results')
        try:
            with open(result_files[metric], 'r') as f:
                predictions = json.load(f)
            coco_dt = _coco_api.loadRes(predictions)
        except IndexError:
            print('The testing results of the whole dataset is empty.')
            break
        
        coco_eval = COCOeval(_coco_api, coco_dt, iou_type)
        
        coco_eval.params.catIds = cat_ids
        coco_eval.params.imgIds = img_ids
        coco_eval.params.maxDets = list(proposal_nums)
        coco_eval.params.iouThrs = iou_thrs
        
        # mapping of cocoEval.stats
        coco_metric_names = {
            'mAP': 0,
            'mAP_50': 1,
            'mAP_75': 2,
            'mAP_s': 3,
            'mAP_m': 4,
            'mAP_l': 5,
            'AR@100': 6,
            'AR@300': 7,
            'AR@1000': 8,
            'AR_s@1000': 9,
            'AR_m@1000': 10,
            'AR_l@1000': 11
        }
        
        coco_eval.evaluate()
        coco_eval.accumulate()
        coco_eval.summarize()
        
        metric_items = [
            'mAP', 'mAP_50', 'mAP_75', 'mAP_s', 'mAP_m', 'mAP_l'
        ]
        
        for metric_item in metric_items:
            val = coco_eval.stats[coco_metric_names[metric_item]]
            eval_results[metric_item] = float(f'{round(val, 3)}')
    
    if tmp_dir is not None:
        tmp_dir.cleanup()
    
    return eval_results