val.py

# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import os
from collections import OrderedDict

import numpy as np
import time
import paddle
import paddle.nn.functional as F
from paddleseg.utils import TimeAverager, calculate_eta, logger, progbar

from utils.evaluation import SemanticEvaluator, InstanceEvaluator, PanopticEvaluator
from core import infer

np.set_printoptions(suppress=True)


def evaluate(model,
             eval_dataset,
             threshold=0.1,
             nms_kernel=7,
             top_k=200,
             num_workers=0,
             print_detail=True):
    """
    Launch evaluation.

    Args:
        model（nn.Layer): A sementic segmentation model.
        eval_dataset (paddle.io.Dataset): Used to read and process validation datasets.
        threshold (float, optional): Threshold applied to center heatmap score. Defalut: 0.1.
        nms_kernel (int, optional): NMS max pooling kernel size. Default: 7.
        top_k (int, optional): Top k centers to keep. Default: 200.
        num_workers (int, optional): Num workers for data loader. Default: 0.
        print_detail (bool, optional): Whether to print detailed information about the evaluation process. Default: True.

    Returns:
        dict: Panoptic evaluation results which includes PQ, RQ, SQ for all, each class, Things and stuff.
        dict: Semantic evaluation results which includes mIoU, fwIoU, mACC and pACC.
        dict: Instance evaluation results which includes mAP and mAP50, and also AP and AP50 for each class.

    """
    model.eval()
    nranks = paddle.distributed.ParallelEnv().nranks
    local_rank = paddle.distributed.ParallelEnv().local_rank
    batch_sampler = paddle.io.BatchSampler(
        eval_dataset, batch_size=1, shuffle=False, drop_last=False)
    loader = paddle.io.DataLoader(
        eval_dataset,
        batch_sampler=batch_sampler,
        num_workers=num_workers,
        return_list=True, )

    total_iters = len(loader)
    semantic_metric = SemanticEvaluator(
        eval_dataset.num_classes, ignore_index=eval_dataset.ignore_index)
    instance_metric_AP50 = InstanceEvaluator(
        eval_dataset.num_classes,
        overlaps=0.5,
        thing_list=eval_dataset.thing_list)
    instance_metric_AP = InstanceEvaluator(
        eval_dataset.num_classes,
        overlaps=list(np.arange(0.5, 1.0, 0.05)),
        thing_list=eval_dataset.thing_list)
    panoptic_metric = PanopticEvaluator(
        num_classes=eval_dataset.num_classes,
        thing_list=eval_dataset.thing_list,
        ignore_index=eval_dataset.ignore_index,
        label_divisor=eval_dataset.label_divisor)

    if print_detail:
        logger.info("Start evaluating (total_samples={}, total_iters={})...".
                    format(len(eval_dataset), total_iters))
    progbar_val = progbar.Progbar(target=total_iters, verbose=1)
    reader_cost_averager = TimeAverager()
    batch_cost_averager = TimeAverager()
    batch_start = time.time()
    with paddle.no_grad():
        for iter, data in enumerate(loader):
            reader_cost_averager.record(time.time() - batch_start)
            im = data[0]
            raw_semantic_label = data[1]  # raw semantic label.
            raw_instance_label = data[2]
            raw_panoptic_label = data[3]
            ori_shape = raw_semantic_label.shape[-2:]

            semantic, semantic_softmax, instance, panoptic, ctr_hmp = infer.inference(
                model=model,
                im=im,
                transforms=eval_dataset.transforms.transforms,
                thing_list=eval_dataset.thing_list,
                label_divisor=eval_dataset.label_divisor,
                stuff_area=eval_dataset.stuff_area,
                ignore_index=eval_dataset.ignore_index,
                threshold=threshold,
                nms_kernel=nms_kernel,
                top_k=top_k,
                ori_shape=ori_shape)
            semantic = semantic.squeeze().numpy()
            semantic_softmax = semantic_softmax.squeeze().numpy()
            instance = instance.squeeze().numpy()
            panoptic = panoptic.squeeze().numpy()
            ctr_hmp = ctr_hmp.squeeze().numpy()
            raw_semantic_label = raw_semantic_label.squeeze().numpy()
            raw_instance_label = raw_instance_label.squeeze().numpy()
            raw_panoptic_label = raw_panoptic_label.squeeze().numpy()

            # update metric for semantic, instance, panoptic
            semantic_metric.update(semantic, raw_semantic_label)

            gts = instance_metric_AP.convert_gt_map(raw_semantic_label,
                                                    raw_instance_label)
            # print([i[0] for i in gts])
            preds = instance_metric_AP.convert_pred_map(semantic_softmax,
                                                        panoptic)
            # print([(i[0], i[1]) for i in preds ])
            ignore_mask = raw_semantic_label == eval_dataset.ignore_index
            instance_metric_AP.update(preds, gts, ignore_mask=ignore_mask)
            instance_metric_AP50.update(preds, gts, ignore_mask=ignore_mask)

            panoptic_metric.update(panoptic, raw_panoptic_label)

            batch_cost_averager.record(
                time.time() - batch_start, num_samples=len(im))
            batch_cost = batch_cost_averager.get_average()
            reader_cost = reader_cost_averager.get_average()

            if local_rank == 0:
                progbar_val.update(iter + 1, [('batch_cost', batch_cost),
                                              ('reader cost', reader_cost)])
            reader_cost_averager.reset()
            batch_cost_averager.reset()
            batch_start = time.time()

    semantic_results = semantic_metric.evaluate()
    panoptic_results = panoptic_metric.evaluate()
    instance_results = OrderedDict()
    ins_ap = instance_metric_AP.evaluate()
    ins_ap50 = instance_metric_AP50.evaluate()
    instance_results['ins_seg'] = OrderedDict()
    instance_results['ins_seg']['mAP'] = ins_ap['ins_seg']['mAP']
    instance_results['ins_seg']['AP'] = ins_ap['ins_seg']['AP']
    instance_results['ins_seg']['mAP50'] = ins_ap50['ins_seg']['mAP']
    instance_results['ins_seg']['AP50'] = ins_ap50['ins_seg']['AP']

    if print_detail:
        logger.info(panoptic_results)
        print()
        logger.info(semantic_results)
        print()
        logger.info(instance_results)
        print()

        pq = panoptic_results['pan_seg']['All']['pq']
        miou = semantic_results['sem_seg']['mIoU']
        map = instance_results['ins_seg']['mAP']
        map50 = instance_results['ins_seg']['mAP50']
        logger.info("PQ: {:.4f}, mIoU: {:.4f}, mAP: {:.4f}, mAP50: {:.4f}".
                    format(pq, miou, map, map50))

    return panoptic_results, semantic_results, instance_results