pipeline_ddpm.py 4.84 KB
Newer Older
Patrick von Platen's avatar
Patrick von Platen committed
1
# Copyright 2023 The HuggingFace Team. All rights reserved.
Patrick von Platen's avatar
Patrick von Platen committed
2
3
4
5
6
7
8
9
10
11
12
13
14
15
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.


16
from typing import List, Optional, Tuple, Union
Pedro Cuenca's avatar
Pedro Cuenca committed
17

Patrick von Platen's avatar
Patrick von Platen committed
18
19
import torch

Dhruv Nair's avatar
Dhruv Nair committed
20
from ...utils.torch_utils import randn_tensor
21
from ..pipeline_utils import DiffusionPipeline, ImagePipelineOutput
Patrick von Platen's avatar
Patrick von Platen committed
22
23


Patrick von Platen's avatar
Patrick von Platen committed
24
class DDPMPipeline(DiffusionPipeline):
25
    r"""
26
27
28
29
    Pipeline for image generation.

    This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods
    implemented for all pipelines (downloading, saving, running on a particular device, etc.).
30
31

    Parameters:
32
33
        unet ([`UNet2DModel`]):
            A `UNet2DModel` to denoise the encoded image latents.
34
35
36
37
        scheduler ([`SchedulerMixin`]):
            A scheduler to be used in combination with `unet` to denoise the encoded image. Can be one of
            [`DDPMScheduler`], or [`DDIMScheduler`].
    """
38

39
    model_cpu_offload_seq = "unet"
40

41
    def __init__(self, unet, scheduler):
Patrick von Platen's avatar
Patrick von Platen committed
42
        super().__init__()
43
        self.register_modules(unet=unet, scheduler=scheduler)
Patrick von Platen's avatar
Patrick von Platen committed
44

Patrick von Platen's avatar
Patrick von Platen committed
45
    @torch.no_grad()
46
    def __call__(
Sid Sahai's avatar
Sid Sahai committed
47
48
        self,
        batch_size: int = 1,
49
        generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None,
50
        num_inference_steps: int = 1000,
Sid Sahai's avatar
Sid Sahai committed
51
52
        output_type: Optional[str] = "pil",
        return_dict: bool = True,
53
    ) -> Union[ImagePipelineOutput, Tuple]:
54
        r"""
55
56
        The call function to the pipeline for generation.

57
        Args:
58
            batch_size (`int`, *optional*, defaults to 1):
59
                The number of images to generate.
60
            generator (`torch.Generator`, *optional*):
61
62
                A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make
                generation deterministic.
63
64
65
            num_inference_steps (`int`, *optional*, defaults to 1000):
                The number of denoising steps. More denoising steps usually lead to a higher quality image at the
                expense of slower inference.
66
            output_type (`str`, *optional*, defaults to `"pil"`):
67
                The output format of the generated image. Choose between `PIL.Image` or `np.array`.
68
            return_dict (`bool`, *optional*, defaults to `True`):
69
                Whether or not to return a [`~pipelines.ImagePipelineOutput`] instead of a plain tuple.
70

71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
        Example:

        ```py
        >>> from diffusers import DDPMPipeline

        >>> # load model and scheduler
        >>> pipe = DDPMPipeline.from_pretrained("google/ddpm-cat-256")

        >>> # run pipeline in inference (sample random noise and denoise)
        >>> image = pipe().images[0]

        >>> # save image
        >>> image.save("ddpm_generated_image.png")
        ```

86
        Returns:
87
88
89
            [`~pipelines.ImagePipelineOutput`] or `tuple`:
                If `return_dict` is `True`, [`~pipelines.ImagePipelineOutput`] is returned, otherwise a `tuple` is
                returned where the first element is a list with the generated images
90
        """
Patrick von Platen's avatar
Patrick von Platen committed
91
        # Sample gaussian noise to begin loop
92
93
94
95
96
97
98
        if isinstance(self.unet.config.sample_size, int):
            image_shape = (
                batch_size,
                self.unet.config.in_channels,
                self.unet.config.sample_size,
                self.unet.config.sample_size,
            )
99
        else:
100
            image_shape = (batch_size, self.unet.config.in_channels, *self.unet.config.sample_size)
101

102
103
        if self.device.type == "mps":
            # randn does not work reproducibly on mps
104
            image = randn_tensor(image_shape, generator=generator)
105
106
            image = image.to(self.device)
        else:
107
            image = randn_tensor(image_shape, generator=generator, device=self.device)
Patrick von Platen's avatar
Patrick von Platen committed
108

109
        # set step values
110
        self.scheduler.set_timesteps(num_inference_steps)
111

hysts's avatar
hysts committed
112
        for t in self.progress_bar(self.scheduler.timesteps):
Patrick von Platen's avatar
Patrick von Platen committed
113
            # 1. predict noise model_output
114
            model_output = self.unet(image, t).sample
Patrick von Platen's avatar
Patrick von Platen committed
115

116
            # 2. compute previous image: x_t -> x_t-1
117
            image = self.scheduler.step(model_output, t, image, generator=generator).prev_sample
Patrick von Platen's avatar
Patrick von Platen committed
118

119
120
        image = (image / 2 + 0.5).clamp(0, 1)
        image = image.cpu().permute(0, 2, 3, 1).numpy()
121
122
        if output_type == "pil":
            image = self.numpy_to_pil(image)
123

124
125
126
127
        if not return_dict:
            return (image,)

        return ImagePipelineOutput(images=image)