# Copyright (c) OpenMMLab. All rights reserved. import functools import torch.nn.functional as F def reduce_loss(loss, reduction): """Reduce loss as specified. Args: loss (Tensor): Elementwise loss tensor. reduction (str): Options are "none", "mean", "sum", "flatmean" and "batchmean". 'none': no reduction will be applied. 'mean': the output will be divided by the number of elements in the output. 'sum': the output will be summed. 'batchmean': the sum of the output will be divided by batchsize. 'flatmean': each sample will be divided by the number of element respectively and output will shape as [bz, ]. Return: Tensor: Reduced loss tensor. """ if reduction == 'batchmean': return loss.sum() / loss.shape[0] if reduction == 'flatmean': return loss.mean(dim=list(range(1, loss.ndim))) reduction_enum = F._Reduction.get_enum(reduction) # none: 0, elementwise_mean:1, sum: 2 if reduction_enum == 0: return loss if reduction_enum == 1: return loss.mean() if reduction_enum == 2: return loss.sum() raise ValueError(f'reduction type {reduction} not supported') def weight_reduce_loss(loss, weight=None, reduction='mean', avg_factor=None): """Apply element-wise weight and reduce loss. Args: loss (Tensor): Element-wise loss. weight (Tensor): Element-wise weights. reduction (str): Same as built-in losses of PyTorch. avg_factor (float): Average factor when computing the mean of losses. Returns: Tensor: Processed loss values. """ # if weight is specified, apply element-wise weight if weight is not None: loss = loss * weight # if avg_factor is not specified, just reduce the loss if avg_factor is None: loss = reduce_loss(loss, reduction) else: # if reduction is mean, then average the loss by avg_factor if reduction == 'mean': loss = loss.sum() / avg_factor # if reduction is 'none', then do nothing, otherwise raise an error elif reduction != 'none': raise ValueError('avg_factor can not be used with reduction="sum"') return loss def weighted_loss(loss_func): """Create a weighted version of a given loss function. To use this decorator, the loss function must have the signature like `loss_func(pred, target, **kwargs)`. The function only needs to compute element-wise loss without any reduction. This decorator will add weight and reduction arguments to the function. The decorated function will have the signature like `loss_func(pred, target, weight=None, reduction='mean', avg_factor=None, **kwargs)`. :Example: >>> import torch >>> @weighted_loss >>> def l1_loss(pred, target): >>> return (pred - target).abs() >>> pred = torch.Tensor([0, 2, 3]) >>> target = torch.Tensor([1, 1, 1]) >>> weight = torch.Tensor([1, 0, 1]) >>> l1_loss(pred, target) tensor(1.3333) >>> l1_loss(pred, target, weight) tensor(1.) >>> l1_loss(pred, target, reduction='none') tensor([1., 1., 2.]) >>> l1_loss(pred, target, weight, avg_factor=2) tensor(1.5000) """ @functools.wraps(loss_func) def wrapper(*args, weight=None, reduction='mean', avg_factor=None, **kwargs): # get element-wise loss loss = loss_func(*args, **kwargs) loss = weight_reduce_loss(loss, weight, reduction, avg_factor) return loss return wrapper