utils.py 10.7 KB
Newer Older
1
from typing import Union, Optional, List, Tuple, Text, BinaryIO
2
import pathlib
3
4
import torch
import math
5
import warnings
6
import numpy as np
7
from PIL import Image, ImageDraw, ImageFont, ImageColor
8

9
__all__ = ["make_grid", "save_image", "draw_bounding_boxes", "draw_segmentation_masks"]
10

11

12
@torch.no_grad()
13
def make_grid(
14
    tensor: Union[torch.Tensor, List[torch.Tensor]],
15
16
17
    nrow: int = 8,
    padding: int = 2,
    normalize: bool = False,
18
    value_range: Optional[Tuple[int, int]] = None,
19
20
    scale_each: bool = False,
    pad_value: int = 0,
21
    **kwargs
22
) -> torch.Tensor:
23
    """Make a grid of images.
24

25
26
27
    Args:
        tensor (Tensor or list): 4D mini-batch Tensor of shape (B x C x H x W)
            or a list of images all of the same size.
28
        nrow (int, optional): Number of images displayed in each row of the grid.
Tongzhou Wang's avatar
Tongzhou Wang committed
29
30
            The final grid size is ``(B / nrow, nrow)``. Default: ``8``.
        padding (int, optional): amount of padding. Default: ``2``.
31
        normalize (bool, optional): If True, shift the image to the range (0, 1),
Tongzhou Wang's avatar
Tongzhou Wang committed
32
            by the min and max values specified by :attr:`range`. Default: ``False``.
33
        value_range (tuple, optional): tuple (min, max) where min and max are numbers,
34
35
            then these numbers are used to normalize the image. By default, min and max
            are computed from the tensor.
Tongzhou Wang's avatar
Tongzhou Wang committed
36
37
38
        scale_each (bool, optional): If ``True``, scale each image in the batch of
            images separately rather than the (min, max) over all images. Default: ``False``.
        pad_value (float, optional): Value for the padded pixels. Default: ``0``.
39

40
41
    Example:
        See this notebook `here <https://gist.github.com/anonymous/bf16430f7750c023141c562f3e9f2a91>`_
42

43
    """
44
45
    if not (torch.is_tensor(tensor) or
            (isinstance(tensor, list) and all(torch.is_tensor(t) for t in tensor))):
46
47
48
49
50
51
        raise TypeError(f'tensor or list of tensors expected, got {type(tensor)}')

    if "range" in kwargs.keys():
        warning = "range will be deprecated, please use value_range instead."
        warnings.warn(warning)
        value_range = kwargs["range"]
52

53
    # if list of tensors, convert to a 4D mini-batch Tensor
54
    if isinstance(tensor, list):
55
        tensor = torch.stack(tensor, dim=0)
56

57
    if tensor.dim() == 2:  # single image H x W
58
        tensor = tensor.unsqueeze(0)
59
    if tensor.dim() == 3:  # single image
60
        if tensor.size(0) == 1:  # if single-channel, convert to 3-channel
Adam Lerer's avatar
Adam Lerer committed
61
            tensor = torch.cat((tensor, tensor, tensor), 0)
62
        tensor = tensor.unsqueeze(0)
63

64
    if tensor.dim() == 4 and tensor.size(1) == 1:  # single-channel images
65
        tensor = torch.cat((tensor, tensor, tensor), 1)
66
67

    if normalize is True:
68
        tensor = tensor.clone()  # avoid modifying tensor in-place
69
70
71
        if value_range is not None:
            assert isinstance(value_range, tuple), \
                "value_range has to be a tuple (min, max) if specified. min and max are numbers"
72

73
74
75
        def norm_ip(img, low, high):
            img.clamp_(min=low, max=high)
            img.sub_(low).div_(max(high - low, 1e-5))
76

77
78
79
        def norm_range(t, value_range):
            if value_range is not None:
                norm_ip(t, value_range[0], value_range[1])
80
            else:
81
                norm_ip(t, float(t.min()), float(t.max()))
82
83
84

        if scale_each is True:
            for t in tensor:  # loop over mini-batch dimension
85
                norm_range(t, value_range)
86
        else:
87
            norm_range(tensor, value_range)
88

89
    if tensor.size(0) == 1:
90
        return tensor.squeeze(0)
91

92
93
94
    # make the mini-batch of images into a grid
    nmaps = tensor.size(0)
    xmaps = min(nrow, nmaps)
95
    ymaps = int(math.ceil(float(nmaps) / xmaps))
96
    height, width = int(tensor.size(2) + padding), int(tensor.size(3) + padding)
97
98
    num_channels = tensor.size(1)
    grid = tensor.new_full((num_channels, height * ymaps + padding, width * xmaps + padding), pad_value)
99
    k = 0
100
101
    for y in range(ymaps):
        for x in range(xmaps):
102
103
            if k >= nmaps:
                break
104
105
106
107
108
            # Tensor.copy_() is a valid method but seems to be missing from the stubs
            # https://pytorch.org/docs/stable/tensors.html#torch.Tensor.copy_
            grid.narrow(1, y * height + padding, height - padding).narrow(  # type: ignore[attr-defined]
                2, x * width + padding, width - padding
            ).copy_(tensor[k])
109
110
111
112
            k = k + 1
    return grid


113
@torch.no_grad()
114
def save_image(
115
    tensor: Union[torch.Tensor, List[torch.Tensor]],
116
117
    fp: Union[Text, pathlib.Path, BinaryIO],
    format: Optional[str] = None,
118
    **kwargs
119
) -> None:
120
121
122
123
124
    """Save a given Tensor into an image file.

    Args:
        tensor (Tensor or list): Image to be saved. If given a mini-batch tensor,
            saves the tensor as a grid of images by calling ``make_grid``.
125
        fp (string or file object): A filename or a file object
126
127
        format(Optional):  If omitted, the format to use is determined from the filename extension.
            If a file object was used instead of a filename, this parameter should always be used.
128
        **kwargs: Other arguments are documented in ``make_grid``.
129
    """
130
131

    grid = make_grid(tensor, **kwargs)
132
    # Add 0.5 after unnormalizing to [0, 255] to round to nearest integer
133
    ndarr = grid.mul(255).add_(0.5).clamp_(0, 255).permute(1, 2, 0).to('cpu', torch.uint8).numpy()
134
    im = Image.fromarray(ndarr)
135
    im.save(fp, format=format)
136
137
138
139
140
141
142
143


@torch.no_grad()
def draw_bounding_boxes(
    image: torch.Tensor,
    boxes: torch.Tensor,
    labels: Optional[List[str]] = None,
    colors: Optional[List[Union[str, Tuple[int, int, int]]]] = None,
144
    fill: Optional[bool] = False,
145
146
147
148
149
150
151
152
    width: int = 1,
    font: Optional[str] = None,
    font_size: int = 10
) -> torch.Tensor:

    """
    Draws bounding boxes on given image.
    The values of the input image should be uint8 between 0 and 255.
153
    If filled, Resulting Tensor should be saved as PNG image.
154
155

    Args:
156
        image (Tensor): Tensor of shape (C x H x W) and dtype uint8.
157
        boxes (Tensor): Tensor of size (N, 4) containing bounding boxes in (xmin, ymin, xmax, ymax) format. Note that
158
159
160
161
162
            the boxes are absolute coordinates with respect to the image. In other words: `0 <= xmin < xmax < W` and
            `0 <= ymin < ymax < H`.
        labels (List[str]): List containing the labels of bounding boxes.
        colors (List[Union[str, Tuple[int, int, int]]]): List containing the colors of bounding boxes. The colors can
            be represented as `str` or `Tuple[int, int, int]`.
163
        fill (bool): If `True` fills the bounding box with specified color.
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
        width (int): Width of bounding box.
        font (str): A filename containing a TrueType font. If the file is not found in this filename, the loader may
            also search in other directories, such as the `fonts/` directory on Windows or `/Library/Fonts/`,
            `/System/Library/Fonts/` and `~/Library/Fonts/` on macOS.
        font_size (int): The requested font size in points.
    """

    if not isinstance(image, torch.Tensor):
        raise TypeError(f"Tensor expected, got {type(image)}")
    elif image.dtype != torch.uint8:
        raise ValueError(f"Tensor uint8 expected, got {image.dtype}")
    elif image.dim() != 3:
        raise ValueError("Pass individual images, not batches")

    ndarr = image.permute(1, 2, 0).numpy()
    img_to_draw = Image.fromarray(ndarr)

    img_boxes = boxes.to(torch.int64).tolist()

183
184
185
186
187
188
    if fill:
        draw = ImageDraw.Draw(img_to_draw, "RGBA")

    else:
        draw = ImageDraw.Draw(img_to_draw)

189
    txt_font = ImageFont.load_default() if font is None else ImageFont.truetype(font=font, size=font_size)
190
191

    for i, bbox in enumerate(img_boxes):
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
        if colors is None:
            color = None
        else:
            color = colors[i]

        if fill:
            if color is None:
                fill_color = (255, 255, 255, 100)
            elif isinstance(color, str):
                # This will automatically raise Error if rgb cannot be parsed.
                fill_color = ImageColor.getrgb(color) + (100,)
            elif isinstance(color, tuple):
                fill_color = color + (100,)
            draw.rectangle(bbox, width=width, outline=color, fill=fill_color)
        else:
            draw.rectangle(bbox, width=width, outline=color)
208
209
210
211
212

        if labels is not None:
            draw.text((bbox[0], bbox[1]), labels[i], fill=color, font=txt_font)

    return torch.from_numpy(np.array(img_to_draw)).permute(2, 0, 1)
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270


@torch.no_grad()
def draw_segmentation_masks(
    image: torch.Tensor,
    masks: torch.Tensor,
    alpha: float = 0.2,
    colors: Optional[List[Union[str, Tuple[int, int, int]]]] = None,
) -> torch.Tensor:

    """
    Draws segmentation masks on given RGB image.
    The values of the input image should be uint8 between 0 and 255.

    Args:
        image (Tensor): Tensor of shape (3 x H x W) and dtype uint8.
        masks (Tensor): Tensor of shape (num_masks, H, W). Each containing probability of predicted class.
        alpha (float): Float number between 0 and 1 denoting factor of transpaerency of masks.
        colors (List[Union[str, Tuple[int, int, int]]]): List containing the colors of masks. The colors can
            be represented as `str` or `Tuple[int, int, int]`.
    """

    if not isinstance(image, torch.Tensor):
        raise TypeError(f"Tensor expected, got {type(image)}")
    elif image.dtype != torch.uint8:
        raise ValueError(f"Tensor uint8 expected, got {image.dtype}")
    elif image.dim() != 3:
        raise ValueError("Pass individual images, not batches")
    elif image.size()[0] != 3:
        raise ValueError("Pass an RGB image. Other Image formats are not supported")

    num_masks = masks.size()[0]
    masks = masks.argmax(0)

    if colors is None:
        palette = torch.tensor([2 ** 25 - 1, 2 ** 15 - 1, 2 ** 21 - 1])
        colors_t = torch.as_tensor([i for i in range(num_masks)])[:, None] * palette
        color_arr = (colors_t % 255).numpy().astype("uint8")
    else:
        color_list = []
        for color in colors:
            if isinstance(color, str):
                # This will automatically raise Error if rgb cannot be parsed.
                fill_color = ImageColor.getrgb(color)
                color_list.append(fill_color)
            elif isinstance(color, tuple):
                color_list.append(color)

        color_arr = np.array(color_list).astype("uint8")

    _, h, w = image.size()
    img_to_draw = Image.fromarray(masks.byte().cpu().numpy()).resize((w, h))
    img_to_draw.putpalette(color_arr)

    img_to_draw = torch.from_numpy(np.array(img_to_draw.convert('RGB')))
    img_to_draw = img_to_draw.permute((2, 0, 1))

    return (image.float() * alpha + img_to_draw.float() * (1.0 - alpha)).to(dtype=torch.uint8)