val.py

# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import os
import time
import tqdm
import numpy as np
from PIL import Image
import paddle
import paddle.nn.functional as F

from paddleseg.utils import metrics, TimeAverager, calculate_eta, logger, progbar
import datasets
from utils import save_imgs

np.set_printoptions(suppress=True)

np.seterr(divide='ignore', invalid='ignore')

# Names
name_classes = [
    'road', 'sidewalk', 'building', 'wall', 'fence', 'pole', 'trafflight',
    'traffsign', 'vegetation', 'terrain', 'sky', 'person', 'rider', 'car',
    'truck', 'bus', 'train', 'motorcycle', 'bicycle', 'unlabeled'
]

synthia_set_16 = [0, 1, 2, 3, 4, 5, 6, 7, 8, 10, 11, 12, 13, 15, 17, 18]
synthia_set_13 = [0, 1, 2, 6, 7, 8, 10, 11, 12, 13, 15, 17, 18]
synthia_set_16_to_13 = [0, 1, 2, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]


class Eval():
    def __init__(self, num_class):
        self.num_class = num_class
        self.confusion_matrix = np.zeros((self.num_class, ) * 2)
        self.ignore_index = None
        self.synthia = True if num_class == 16 else False

    def pixel_accuracy(self):
        if np.sum(self.confusion_matrix) == 0:
            print("Attention: pixel_total is zero!!!")
            PA = 0
        else:
            PA = np.diag(self.confusion_matrix).sum(
            ) / self.confusion_matrix.sum()

        return PA

    def mean_pixel_accuracy(self, out_16_13=False):
        MPA = np.diag(self.confusion_matrix) / self.confusion_matrix.sum(axis=1)
        if self.synthia:
            MPA_16 = np.nanmean(MPA[:self.ignore_index])
            MPA_13 = np.nanmean(MPA[synthia_set_16_to_13])
            return MPA_16, MPA_13
        if out_16_13:
            MPA_16 = np.nanmean(MPA[synthia_set_16])
            MPA_13 = np.nanmean(MPA[synthia_set_13])
            return MPA_16, MPA_13
        MPA = np.nanmean(MPA[:self.ignore_index])

        return MPA

    def mean_iou(self, out_16_13=False):
        MIoU = np.diag(self.confusion_matrix) / (
            np.sum(self.confusion_matrix, axis=1) + np.sum(
                self.confusion_matrix, axis=0) - np.diag(self.confusion_matrix))
        if self.synthia:
            MIoU_16 = np.nanmean(MIoU[:self.ignore_index])
            MIoU_13 = np.nanmean(MIoU[synthia_set_16_to_13])
            return MIoU_16, MIoU_13
        if out_16_13:
            MIoU_16 = np.nanmean(MIoU[synthia_set_16])
            MIoU_13 = np.nanmean(MIoU[synthia_set_13])
            return MIoU_16, MIoU_13
        MIoU = np.nanmean(MIoU[:self.ignore_index])

        return MIoU

    def fwiou(self, out_16_13=False):
        FWIoU = np.multiply(
            np.sum(self.confusion_matrix, axis=1),
            np.diag(self.confusion_matrix))
        FWIoU = FWIoU / (np.sum(self.confusion_matrix, axis=1) + np.sum(
            self.confusion_matrix, axis=0) - np.diag(self.confusion_matrix))
        if self.synthia:
            FWIoU_16 = np.sum(
                i for i in FWIoU
                if not np.isnan(i)) / np.sum(self.confusion_matrix)
            FWIoU_13 = np.sum(
                i for i in FWIoU[synthia_set_16_to_13]
                if not np.isnan(i)) / np.sum(self.confusion_matrix)
            return FWIoU_16, FWIoU_13
        if out_16_13:
            FWIoU_16 = np.sum(
                i for i in FWIoU[synthia_set_16]
                if not np.isnan(i)) / np.sum(self.confusion_matrix)
            FWIoU_13 = np.sum(
                i for i in FWIoU[synthia_set_13]
                if not np.isnan(i)) / np.sum(self.confusion_matrix)
            return FWIoU_16, FWIoU_13
        FWIoU = np.sum(i for i in FWIoU
                       if not np.isnan(i)) / np.sum(self.confusion_matrix)

        return FWIoU

    def mean_precision(self, out_16_13=False):
        Precision = np.diag(self.confusion_matrix) / self.confusion_matrix.sum(
            axis=0)
        if self.synthia:
            Precision_16 = np.nanmean(Precision[:self.ignore_index])
            Precision_13 = np.nanmean(Precision[synthia_set_16_to_13])
            return Precision_16, Precision_13
        if out_16_13:
            Precision_16 = np.nanmean(Precision[synthia_set_16])
            Precision_13 = np.nanmean(Precision[synthia_set_13])
            return Precision_16, Precision_13
        Precision = np.nanmean(Precision[:self.ignore_index])
        return Precision

    def print_every_class_eval(self, out_16_13=False, logger=None):
        MIoU = np.diag(self.confusion_matrix) / (
            np.sum(self.confusion_matrix, axis=1) + np.sum(
                self.confusion_matrix, axis=0) - np.diag(self.confusion_matrix))
        MPA = np.diag(self.confusion_matrix) / self.confusion_matrix.sum(axis=1)
        Precision = np.diag(self.confusion_matrix) / self.confusion_matrix.sum(
            axis=0)
        Class_ratio = np.sum(self.confusion_matrix,
                             axis=1) / np.sum(self.confusion_matrix)
        Pred_retio = np.sum(self.confusion_matrix,
                            axis=0) / np.sum(self.confusion_matrix)
        log_fn = print if logger is None else logger.info
        log_fn('===>Everyclass:\t' + 'MPA\t' + 'MIoU\t' + 'PC\t' + 'Ratio\t' +
               'Pred_Retio')
        # if out_16_13: MIoU = MIoU[synthia_set_16]
        for ind_class in range(len(MIoU)):
            pa = str(round(MPA[ind_class] * 100, 2)) if not np.isnan(MPA[
                ind_class]) else 'nan'
            iou = str(round(MIoU[ind_class] * 100, 2)) if not np.isnan(MIoU[
                ind_class]) else 'nan'
            pc = str(round(Precision[ind_class] * 100, 2)) if not np.isnan(
                Precision[ind_class]) else 'nan'
            cr = str(round(Class_ratio[ind_class] * 100, 2)) if not np.isnan(
                Class_ratio[ind_class]) else 'nan'
            pr = str(round(Pred_retio[ind_class] * 100, 2)) if not np.isnan(
                Pred_retio[ind_class]) else 'nan'
            log_fn('===>' + name_classes[ind_class] + ':\t' + pa + '\t' + iou +
                   '\t' + pc + '\t' + cr + '\t' + pr)

    # generate confusion matrix
    def __generate_matrix(self, gt_image, pre_image):

        mask = (gt_image >= 0) & (gt_image < self.num_class)
        label = self.num_class * gt_image[mask].astype('int') + pre_image[mask]
        count = np.bincount(label, minlength=self.num_class**2)
        confusion_matrix = count.reshape(self.num_class, self.num_class)
        return confusion_matrix

    def add_batch(self, gt_image, pre_image):
        # assert the size of two images are same
        assert gt_image.shape == pre_image.shape
        self.confusion_matrix += self.__generate_matrix(gt_image, pre_image)

    def reset(self):
        self.confusion_matrix = np.zeros((self.num_class, ) * 2)


def evaluate(model,
             eval_dataset,
             num_workers=0,
             print_detail=True,
             save_img=True):
    """
    Launch evalution.

    Args:
        model（nn.Layer): A sementic segmentation model.
        eval_dataset (paddle.io.Dataset): Used to read and process validation datasets.
        num_workers (int, optional): Num workers for data loader. Default: 0.
        print_detail (bool, optional): Whether to print detailed information about the evaluation process. Default: True.

    Returns:
        float: The mIoU of validation datasets.
        float: The accuracy of validation datasets.
    """
    logger.info('Validating')
    evaluator = Eval(eval_dataset.NUM_CLASSES)
    evaluator.reset()

    model.eval()

    nranks = paddle.distributed.ParallelEnv().nranks
    local_rank = paddle.distributed.ParallelEnv().local_rank
    if nranks > 1:
        # Initialize parallel environment if not done.
        if not paddle.distributed.parallel.parallel_helper._is_parallel_ctx_initialized(
        ):
            paddle.distributed.init_parallel_env()

    batch_sampler = paddle.io.DistributedBatchSampler(
        eval_dataset, batch_size=1, shuffle=False, drop_last=True)
    loader = paddle.io.DataLoader(
        eval_dataset,
        batch_sampler=batch_sampler,
        num_workers=num_workers,
        return_list=True, )

    progbar_val = progbar.Progbar(
        target=len(loader), verbose=0 if nranks < 2 else 2)

    reader_cost_averager = TimeAverager()
    batch_cost_averager = TimeAverager()
    batch_start = time.time()

    with paddle.no_grad():
        for idx, (x, y, _, item) in enumerate(loader):
            reader_cost_averager.record(time.time() - batch_start)

            # Forward
            y = y.astype('int64')
            pred = model(x)  # 1, c, h, w
            if len(pred) > 1:
                pred = pred[0]

            # Convert to numpy
            label = y.squeeze(axis=1).numpy()  #
            argpred = np.argmax(pred.numpy(), axis=1)  # 1, 1, H, W
            if save_img:
                save_imgs(argpred, item, './output/')

            # Add to evaluator
            evaluator.add_batch(label, argpred)

            batch_cost_averager.record(
                time.time() - batch_start, num_samples=len(label))
            batch_cost = batch_cost_averager.get_average()
            reader_cost = reader_cost_averager.get_average()

            if local_rank == 0 and print_detail and idx % 10 == 0:
                progbar_val.update(idx + 1, [('batch_cost', batch_cost),
                                             ('reader cost', reader_cost)])
            reader_cost_averager.reset()
            batch_cost_averager.reset()
            batch_start = time.time()

        PA = evaluator.pixel_accuracy()
        MPA = evaluator.mean_pixel_accuracy()
        MIoU = evaluator.mean_iou()
        FWIoU = evaluator.fwiou()
        PC = evaluator.mean_precision()
        logger.info(
            'PA1:{:.3f}, MPA1:{:.3f}, MIoU1:{:.3f}, FWIoU1:{:.3f}, PC:{:.3f}'.
            format(PA, MPA, MIoU, FWIoU, PC))

    return PA, MPA, MIoU, FWIoU