# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import os from collections import OrderedDict import numpy as np import time import paddle import paddle.nn.functional as F from paddleseg.utils import TimeAverager, calculate_eta, logger, progbar from utils.evaluation import SemanticEvaluator, InstanceEvaluator, PanopticEvaluator from core import infer np.set_printoptions(suppress=True) def evaluate(model, eval_dataset, threshold=0.1, nms_kernel=7, top_k=200, num_workers=0, print_detail=True): """ Launch evaluation. Args: model（nn.Layer): A sementic segmentation model. eval_dataset (paddle.io.Dataset): Used to read and process validation datasets. threshold (float, optional): Threshold applied to center heatmap score. Defalut: 0.1. nms_kernel (int, optional): NMS max pooling kernel size. Default: 7. top_k (int, optional): Top k centers to keep. Default: 200. num_workers (int, optional): Num workers for data loader. Default: 0. print_detail (bool, optional): Whether to print detailed information about the evaluation process. Default: True. Returns: dict: Panoptic evaluation results which includes PQ, RQ, SQ for all, each class, Things and stuff. dict: Semantic evaluation results which includes mIoU, fwIoU, mACC and pACC. dict: Instance evaluation results which includes mAP and mAP50, and also AP and AP50 for each class. """ model.eval() nranks = paddle.distributed.ParallelEnv().nranks local_rank = paddle.distributed.ParallelEnv().local_rank batch_sampler = paddle.io.BatchSampler( eval_dataset, batch_size=1, shuffle=False, drop_last=False) loader = paddle.io.DataLoader( eval_dataset, batch_sampler=batch_sampler, num_workers=num_workers, return_list=True, ) total_iters = len(loader) semantic_metric = SemanticEvaluator( eval_dataset.num_classes, ignore_index=eval_dataset.ignore_index) instance_metric_AP50 = InstanceEvaluator( eval_dataset.num_classes, overlaps=0.5, thing_list=eval_dataset.thing_list) instance_metric_AP = InstanceEvaluator( eval_dataset.num_classes, overlaps=list(np.arange(0.5, 1.0, 0.05)), thing_list=eval_dataset.thing_list) panoptic_metric = PanopticEvaluator( num_classes=eval_dataset.num_classes, thing_list=eval_dataset.thing_list, ignore_index=eval_dataset.ignore_index, label_divisor=eval_dataset.label_divisor) if print_detail: logger.info("Start evaluating (total_samples={}, total_iters={})...". format(len(eval_dataset), total_iters)) progbar_val = progbar.Progbar(target=total_iters, verbose=1) reader_cost_averager = TimeAverager() batch_cost_averager = TimeAverager() batch_start = time.time() with paddle.no_grad(): for iter, data in enumerate(loader): reader_cost_averager.record(time.time() - batch_start) im = data[0] raw_semantic_label = data[1] # raw semantic label. raw_instance_label = data[2] raw_panoptic_label = data[3] ori_shape = raw_semantic_label.shape[-2:] semantic, semantic_softmax, instance, panoptic, ctr_hmp = infer.inference( model=model, im=im, transforms=eval_dataset.transforms.transforms, thing_list=eval_dataset.thing_list, label_divisor=eval_dataset.label_divisor, stuff_area=eval_dataset.stuff_area, ignore_index=eval_dataset.ignore_index, threshold=threshold, nms_kernel=nms_kernel, top_k=top_k, ori_shape=ori_shape) semantic = semantic.squeeze().numpy() semantic_softmax = semantic_softmax.squeeze().numpy() instance = instance.squeeze().numpy() panoptic = panoptic.squeeze().numpy() ctr_hmp = ctr_hmp.squeeze().numpy() raw_semantic_label = raw_semantic_label.squeeze().numpy() raw_instance_label = raw_instance_label.squeeze().numpy() raw_panoptic_label = raw_panoptic_label.squeeze().numpy() # update metric for semantic, instance, panoptic semantic_metric.update(semantic, raw_semantic_label) gts = instance_metric_AP.convert_gt_map(raw_semantic_label, raw_instance_label) # print([i[0] for i in gts]) preds = instance_metric_AP.convert_pred_map(semantic_softmax, panoptic) # print([(i[0], i[1]) for i in preds ]) ignore_mask = raw_semantic_label == eval_dataset.ignore_index instance_metric_AP.update(preds, gts, ignore_mask=ignore_mask) instance_metric_AP50.update(preds, gts, ignore_mask=ignore_mask) panoptic_metric.update(panoptic, raw_panoptic_label) batch_cost_averager.record( time.time() - batch_start, num_samples=len(im)) batch_cost = batch_cost_averager.get_average() reader_cost = reader_cost_averager.get_average() if local_rank == 0: progbar_val.update(iter + 1, [('batch_cost', batch_cost), ('reader cost', reader_cost)]) reader_cost_averager.reset() batch_cost_averager.reset() batch_start = time.time() semantic_results = semantic_metric.evaluate() panoptic_results = panoptic_metric.evaluate() instance_results = OrderedDict() ins_ap = instance_metric_AP.evaluate() ins_ap50 = instance_metric_AP50.evaluate() instance_results['ins_seg'] = OrderedDict() instance_results['ins_seg']['mAP'] = ins_ap['ins_seg']['mAP'] instance_results['ins_seg']['AP'] = ins_ap['ins_seg']['AP'] instance_results['ins_seg']['mAP50'] = ins_ap50['ins_seg']['mAP'] instance_results['ins_seg']['AP50'] = ins_ap50['ins_seg']['AP'] if print_detail: logger.info(panoptic_results) print() logger.info(semantic_results) print() logger.info(instance_results) print() pq = panoptic_results['pan_seg']['All']['pq'] miou = semantic_results['sem_seg']['mIoU'] map = instance_results['ins_seg']['mAP'] map50 = instance_results['ins_seg']['mAP50'] logger.info("PQ: {:.4f}, mIoU: {:.4f}, mAP: {:.4f}, mAP50: {:.4f}". format(pq, miou, map, map50)) return panoptic_results, semantic_results, instance_results