pipeline_utils.py 18.3 KB
Newer Older
Patrick von Platen's avatar
Patrick von Platen committed
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
# coding=utf-8
# Copyright 2022 The HuggingFace Inc. team.
# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

Patrick von Platen's avatar
improve  
Patrick von Platen committed
17
import importlib
18
import inspect
Patrick von Platen's avatar
Patrick von Platen committed
19
import os
20
21
from dataclasses import dataclass
from typing import List, Optional, Union
anton-l's avatar
Style  
anton-l committed
22

23
import numpy as np
Pedro Cuenca's avatar
Pedro Cuenca committed
24
25
import torch

26
import diffusers
27
import PIL
Patrick von Platen's avatar
up  
Patrick von Platen committed
28
from huggingface_hub import snapshot_download
29
from PIL import Image
hysts's avatar
hysts committed
30
from tqdm.auto import tqdm
Patrick von Platen's avatar
Patrick von Platen committed
31

Patrick von Platen's avatar
Patrick von Platen committed
32
from .configuration_utils import ConfigMixin
33
from .utils import DIFFUSERS_CACHE, BaseOutput, logging
Patrick von Platen's avatar
improve  
Patrick von Platen committed
34

Patrick von Platen's avatar
Patrick von Platen committed
35

Patrick von Platen's avatar
Patrick von Platen committed
36
INDEX_FILE = "diffusion_pytorch_model.bin"
Patrick von Platen's avatar
Patrick von Platen committed
37
38
39
40
41
42
43


logger = logging.get_logger(__name__)


LOADABLE_CLASSES = {
    "diffusers": {
Patrick von Platen's avatar
Patrick von Platen committed
44
        "ModelMixin": ["save_pretrained", "from_pretrained"],
Patrick von Platen's avatar
Patrick von Platen committed
45
        "SchedulerMixin": ["save_config", "from_config"],
Patrick von Platen's avatar
Patrick von Platen committed
46
        "DiffusionPipeline": ["save_pretrained", "from_pretrained"],
47
        "OnnxRuntimeModel": ["save_pretrained", "from_pretrained"],
Patrick von Platen's avatar
Patrick von Platen committed
48
49
    },
    "transformers": {
anton-l's avatar
anton-l committed
50
        "PreTrainedTokenizer": ["save_pretrained", "from_pretrained"],
51
        "PreTrainedTokenizerFast": ["save_pretrained", "from_pretrained"],
anton-l's avatar
anton-l committed
52
        "PreTrainedModel": ["save_pretrained", "from_pretrained"],
Suraj Patil's avatar
Suraj Patil committed
53
        "FeatureExtractionMixin": ["save_pretrained", "from_pretrained"],
Patrick von Platen's avatar
Patrick von Platen committed
54
55
56
    },
}

57
58
59
60
ALL_IMPORTABLE_CLASSES = {}
for library in LOADABLE_CLASSES:
    ALL_IMPORTABLE_CLASSES.update(LOADABLE_CLASSES[library])

Patrick von Platen's avatar
Patrick von Platen committed
61

62
63
64
65
66
67
68
69
70
71
72
73
74
75
@dataclass
class ImagePipelineOutput(BaseOutput):
    """
    Output class for image pipelines.

    Args:
        images (`List[PIL.Image.Image]` or `np.ndarray`)
            List of denoised PIL images of length `batch_size` or numpy array of shape `(batch_size, height, width,
            num_channels)`. PIL images or numpy array present the denoised images of the diffusion pipeline.
    """

    images: Union[List[PIL.Image.Image], np.ndarray]


Patrick von Platen's avatar
Patrick von Platen committed
76
class DiffusionPipeline(ConfigMixin):
77
78
79
80
81
82
83
84
85
86
87
88
    r"""
    Base class for all models.

    [`DiffusionPipeline`] takes care of storing all components (models, schedulers, processors) for diffusion pipelines
    and handles methods for loading, downloading and saving models as well as a few methods common to all pipelines to:

        - move all PyTorch modules to the device of your choice
        - enabling/disabling the progress bar for the denoising iteration

    Class attributes:

        - **config_name** ([`str`]) -- name of the config file that will store the class and module names of all
89
          components of the diffusion pipeline.
90
    """
Patrick von Platen's avatar
Patrick von Platen committed
91
92
    config_name = "model_index.json"

Patrick von Platen's avatar
up  
Patrick von Platen committed
93
    def register_modules(self, **kwargs):
94
95
        # import it here to avoid circular import
        from diffusers import pipelines
96

Patrick von Platen's avatar
Patrick von Platen committed
97
        for name, module in kwargs.items():
98
            # retrieve library
Patrick von Platen's avatar
Patrick von Platen committed
99
            library = module.__module__.split(".")[0]
100

101
102
            # check if the module is a pipeline module
            pipeline_dir = module.__module__.split(".")[-2]
Suraj Patil's avatar
Suraj Patil committed
103
104
            path = module.__module__.split(".")
            is_pipeline_module = pipeline_dir in path and hasattr(pipelines, pipeline_dir)
105

106
107
            # if library is not in LOADABLE_CLASSES, then it is a custom module.
            # Or if it's a pipeline module, then the module is inside the pipeline
108
            # folder so we set the library to module name.
109
            if library not in LOADABLE_CLASSES or is_pipeline_module:
110
                library = pipeline_dir
patil-suraj's avatar
patil-suraj committed
111

112
            # retrieve class_name
Patrick von Platen's avatar
Patrick von Platen committed
113
114
            class_name = module.__class__.__name__

115
116
            register_dict = {name: (library, class_name)}

Patrick von Platen's avatar
Patrick von Platen committed
117
            # save model index config
118
            self.register_to_config(**register_dict)
Patrick von Platen's avatar
Patrick von Platen committed
119
120
121

            # set models
            setattr(self, name, module)
122

Patrick von Platen's avatar
Patrick von Platen committed
123
    def save_pretrained(self, save_directory: Union[str, os.PathLike]):
124
125
126
127
128
129
130
131
132
        """
        Save all variables of the pipeline that can be saved and loaded as well as the pipelines configuration file to
        a directory. A pipeline variable can be saved and loaded if its class implements both a save and loading
        method. The pipeline can easily be re-loaded using the `[`~DiffusionPipeline.from_pretrained`]` class method.

        Arguments:
            save_directory (`str` or `os.PathLike`):
                Directory to which to save. Will be created if it doesn't exist.
        """
Patrick von Platen's avatar
Patrick von Platen committed
133
134
        self.save_config(save_directory)

Patrick von Platen's avatar
Patrick von Platen committed
135
        model_index_dict = dict(self.config)
Patrick von Platen's avatar
Patrick von Platen committed
136
        model_index_dict.pop("_class_name")
137
        model_index_dict.pop("_diffusers_version")
138
        model_index_dict.pop("_module", None)
Patrick von Platen's avatar
Patrick von Platen committed
139

anton-l's avatar
anton-l committed
140
141
142
        for pipeline_component_name in model_index_dict.keys():
            sub_model = getattr(self, pipeline_component_name)
            model_cls = sub_model.__class__
Patrick von Platen's avatar
Patrick von Platen committed
143
144

            save_method_name = None
anton-l's avatar
anton-l committed
145
146
147
148
149
150
151
152
153
154
155
156
157
158
            # search for the model's base class in LOADABLE_CLASSES
            for library_name, library_classes in LOADABLE_CLASSES.items():
                library = importlib.import_module(library_name)
                for base_class, save_load_methods in library_classes.items():
                    class_candidate = getattr(library, base_class)
                    if issubclass(model_cls, class_candidate):
                        # if we found a suitable base class in LOADABLE_CLASSES then grab its save method
                        save_method_name = save_load_methods[0]
                        break
                if save_method_name is not None:
                    break

            save_method = getattr(sub_model, save_method_name)
            save_method(os.path.join(save_directory, pipeline_component_name))
Patrick von Platen's avatar
Patrick von Platen committed
159

Pedro Cuenca's avatar
Pedro Cuenca committed
160
161
162
163
164
165
166
167
168
169
170
171
172
    def to(self, torch_device: Optional[Union[str, torch.device]] = None):
        if torch_device is None:
            return self

        module_names, _ = self.extract_init_dict(dict(self.config))
        for name in module_names.keys():
            module = getattr(self, name)
            if isinstance(module, torch.nn.Module):
                module.to(torch_device)
        return self

    @property
    def device(self) -> torch.device:
173
174
175
176
        r"""
        Returns:
            `torch.device`: The torch device on which the pipeline is located.
        """
Pedro Cuenca's avatar
Pedro Cuenca committed
177
178
179
180
181
182
183
        module_names, _ = self.extract_init_dict(dict(self.config))
        for name in module_names.keys():
            module = getattr(self, name)
            if isinstance(module, torch.nn.Module):
                return module.device
        return torch.device("cpu")

Patrick von Platen's avatar
Patrick von Platen committed
184
185
    @classmethod
    def from_pretrained(cls, pretrained_model_name_or_path: Optional[Union[str, os.PathLike]], **kwargs):
186
        r"""
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
        Instantiate a PyTorch diffusion pipeline from pre-trained pipeline weights.

        The pipeline is set in evaluation mode by default using `model.eval()` (Dropout modules are deactivated).

        The warning *Weights from XXX not initialized from pretrained model* means that the weights of XXX do not come
        pretrained with the rest of the model. It is up to you to train those weights with a downstream fine-tuning
        task.

        The warning *Weights from XXX not used in YYY* means that the layer XXX is not used by YYY, therefore those
        weights are discarded.

        Parameters:
            pretrained_model_name_or_path (`str` or `os.PathLike`, *optional*):
                Can be either:

                    - A string, the *repo id* of a pretrained pipeline hosted inside a model repo on
                      https://huggingface.co/ Valid repo ids have to be located under a user or organization name, like
                      `CompVis/ldm-text2im-large-256`.
                    - A path to a *directory* containing pipeline weights saved using
                      [`~DiffusionPipeline.save_pretrained`], e.g., `./my_pipeline_directory/`.
            torch_dtype (`str` or `torch.dtype`, *optional*):
                Override the default `torch.dtype` and load the model under this dtype. If `"auto"` is passed the dtype
                will be automatically derived from the model's weights.
            force_download (`bool`, *optional*, defaults to `False`):
                Whether or not to force the (re-)download of the model weights and configuration files, overriding the
                cached versions if they exist.
            resume_download (`bool`, *optional*, defaults to `False`):
                Whether or not to delete incompletely received files. Will attempt to resume the download if such a
                file exists.
            proxies (`Dict[str, str]`, *optional*):
                A dictionary of proxy servers to use by protocol or endpoint, e.g., `{'http': 'foo.bar:3128',
                'http://hostname': 'foo.bar:4012'}`. The proxies are used on each request.
            output_loading_info(`bool`, *optional*, defaults to `False`):
220
                Whether or not to also return a dictionary containing missing keys, unexpected keys and error messages.
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
            local_files_only(`bool`, *optional*, defaults to `False`):
                Whether or not to only look at local files (i.e., do not try to download the model).
            use_auth_token (`str` or *bool*, *optional*):
                The token to use as HTTP bearer authorization for remote files. If `True`, will use the token generated
                when running `huggingface-cli login` (stored in `~/.huggingface`).
            revision (`str`, *optional*, defaults to `"main"`):
                The specific model version to use. It can be a branch name, a tag name, or a commit id, since we use a
                git-based system for storing models and other artifacts on huggingface.co, so `revision` can be any
                identifier allowed by git.
            mirror (`str`, *optional*):
                Mirror source to accelerate downloads in China. If you are from China and have an accessibility
                problem, you can set this option to resolve it. Note that we do not guarantee the timeliness or safety.
                Please refer to the mirror site for more information. specify the folder name here.

            kwargs (remaining dictionary of keyword arguments, *optional*):
                Can be used to overwrite load - and saveable variables - *i.e.* the pipeline components - of the
237
                specific pipeline class. The overritten components are then directly passed to the pipelines `__init__`
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
                method. See example below for more information.

        <Tip>

        Passing `use_auth_token=True`` is required when you want to use a private model, *e.g.*
        `"CompVis/stable-diffusion-v1-4"`

        </Tip>

        <Tip>

        Activate the special ["offline-mode"](https://huggingface.co/diffusers/installation.html#offline-mode) to use
        this method in a firewalled environment.

        </Tip>

        Examples:

        ```py
        >>> from diffusers import DiffusionPipeline

        >>> # Download pipeline from huggingface.co and cache.
        >>> pipeline = DiffusionPipeline.from_pretrained("CompVis/ldm-text2im-large-256")

        >>> # Download pipeline that requires an authorization token
        >>> # For more information on access tokens, please refer to this section
        >>> # of the documentation](https://huggingface.co/docs/hub/security-tokens)
        >>> pipeline = DiffusionPipeline.from_pretrained("CompVis/stable-diffusion-v1-4", use_auth_token=True)

        >>> # Download pipeline, but overwrite scheduler
        >>> from diffusers import LMSDiscreteScheduler

        >>> scheduler = LMSDiscreteScheduler(beta_start=0.00085, beta_end=0.012, beta_schedule="scaled_linear")
        >>> pipeline = DiffusionPipeline.from_pretrained(
        ...     "CompVis/stable-diffusion-v1-4", scheduler=scheduler, use_auth_token=True
        ... )
        ```
275
276
277
278
279
280
        """
        cache_dir = kwargs.pop("cache_dir", DIFFUSERS_CACHE)
        resume_download = kwargs.pop("resume_download", False)
        proxies = kwargs.pop("proxies", None)
        local_files_only = kwargs.pop("local_files_only", False)
        use_auth_token = kwargs.pop("use_auth_token", None)
281
        revision = kwargs.pop("revision", None)
282
        torch_dtype = kwargs.pop("torch_dtype", None)
283
        provider = kwargs.pop("provider", None)
Patrick von Platen's avatar
Patrick von Platen committed
284

patil-suraj's avatar
patil-suraj committed
285
        # 1. Download the checkpoints and configs
Patrick von Platen's avatar
Patrick von Platen committed
286
        # use snapshot download here to get it working from from_pretrained
Patrick von Platen's avatar
Patrick von Platen committed
287
        if not os.path.isdir(pretrained_model_name_or_path):
288
289
290
291
292
293
294
            cached_folder = snapshot_download(
                pretrained_model_name_or_path,
                cache_dir=cache_dir,
                resume_download=resume_download,
                proxies=proxies,
                local_files_only=local_files_only,
                use_auth_token=use_auth_token,
295
                revision=revision,
296
            )
Patrick von Platen's avatar
Patrick von Platen committed
297
298
        else:
            cached_folder = pretrained_model_name_or_path
299

patil-suraj's avatar
patil-suraj committed
300
        config_dict = cls.get_config_dict(cached_folder)
301

Patrick von Platen's avatar
Patrick von Platen committed
302
        # 2. Load the pipeline class, if using custom module then load it from the hub
303
304
        # if we load from explicit class, let's use it
        if cls != DiffusionPipeline:
305
306
            pipeline_class = cls
        else:
Patrick von Platen's avatar
Patrick von Platen committed
307
308
309
            diffusers_module = importlib.import_module(cls.__module__.split(".")[0])
            pipeline_class = getattr(diffusers_module, config_dict["_class_name"])

310
311
312
313
314
315
        # some modules can be passed directly to the init
        # in this case they are already instantiated in `kwargs`
        # extract them here
        expected_modules = set(inspect.signature(pipeline_class.__init__).parameters.keys())
        passed_class_obj = {k: kwargs.pop(k) for k in expected_modules if k in kwargs}

316
        init_dict, _ = pipeline_class.extract_init_dict(config_dict, **kwargs)
Patrick von Platen's avatar
Patrick von Platen committed
317
318

        init_kwargs = {}
319

320
321
        # import it here to avoid circular import
        from diffusers import pipelines
322

Patrick von Platen's avatar
Patrick von Platen committed
323
        # 3. Load each module in the pipeline
patil-suraj's avatar
patil-suraj committed
324
        for name, (library_name, class_name) in init_dict.items():
325
            is_pipeline_module = hasattr(pipelines, library_name)
326
327
            loaded_sub_model = None

328
            # if the model is in a pipeline module, then we load it from the pipeline
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
            if name in passed_class_obj:
                # 1. check that passed_class_obj has correct parent class
                if not is_pipeline_module:
                    library = importlib.import_module(library_name)
                    class_obj = getattr(library, class_name)
                    importable_classes = LOADABLE_CLASSES[library_name]
                    class_candidates = {c: getattr(library, c) for c in importable_classes.keys()}

                    expected_class_obj = None
                    for class_name, class_candidate in class_candidates.items():
                        if issubclass(class_obj, class_candidate):
                            expected_class_obj = class_candidate

                    if not issubclass(passed_class_obj[name].__class__, expected_class_obj):
                        raise ValueError(
                            f"{passed_class_obj[name]} is of type: {type(passed_class_obj[name])}, but should be"
                            f" {expected_class_obj}"
                        )
                else:
                    logger.warn(
                        f"You have passed a non-standard module {passed_class_obj[name]}. We cannot verify whether it"
                        " has the correct type"
                    )

                # set passed class object
                loaded_sub_model = passed_class_obj[name]
            elif is_pipeline_module:
356
357
358
                pipeline_module = getattr(pipelines, library_name)
                class_obj = getattr(pipeline_module, class_name)
                importable_classes = ALL_IMPORTABLE_CLASSES
Patrick von Platen's avatar
Patrick von Platen committed
359
                class_candidates = {c: class_obj for c in importable_classes.keys()}
patil-suraj's avatar
patil-suraj committed
360
            else:
patil-suraj's avatar
patil-suraj committed
361
                # else we just import it from the library.
patil-suraj's avatar
patil-suraj committed
362
363
                library = importlib.import_module(library_name)
                class_obj = getattr(library, class_name)
364
                importable_classes = LOADABLE_CLASSES[library_name]
patil-suraj's avatar
patil-suraj committed
365
                class_candidates = {c: getattr(library, c) for c in importable_classes.keys()}
366

367
368
369
370
371
            if loaded_sub_model is None:
                load_method_name = None
                for class_name, class_candidate in class_candidates.items():
                    if issubclass(class_obj, class_candidate):
                        load_method_name = importable_classes[class_name][1]
Patrick von Platen's avatar
Patrick von Platen committed
372

373
                load_method = getattr(class_obj, load_method_name)
Patrick von Platen's avatar
Patrick von Platen committed
374

375
376
377
                loading_kwargs = {}
                if issubclass(class_obj, torch.nn.Module):
                    loading_kwargs["torch_dtype"] = torch_dtype
378
379
                if issubclass(class_obj, diffusers.OnnxRuntimeModel):
                    loading_kwargs["provider"] = provider
380

381
382
                # check if the module is in a subdirectory
                if os.path.isdir(os.path.join(cached_folder, name)):
383
                    loaded_sub_model = load_method(os.path.join(cached_folder, name), **loading_kwargs)
384
385
                else:
                    # else load from the root directory
386
                    loaded_sub_model = load_method(cached_folder, **loading_kwargs)
Patrick von Platen's avatar
Patrick von Platen committed
387

388
            init_kwargs[name] = loaded_sub_model  # UNet(...), # DiffusionSchedule(...)
Patrick von Platen's avatar
Patrick von Platen committed
389

390
        # 4. Instantiate the pipeline
391
        model = pipeline_class(**init_kwargs)
Patrick von Platen's avatar
Patrick von Platen committed
392
        return model
393
394
395
396
397
398
399
400
401
402
403
404

    @staticmethod
    def numpy_to_pil(images):
        """
        Convert a numpy image or a batch of images to a PIL image.
        """
        if images.ndim == 3:
            images = images[None, ...]
        images = (images * 255).round().astype("uint8")
        pil_images = [Image.fromarray(image) for image in images]

        return pil_images
hysts's avatar
hysts committed
405
406
407
408
409
410
411
412
413
414
415
416
417

    def progress_bar(self, iterable):
        if not hasattr(self, "_progress_bar_config"):
            self._progress_bar_config = {}
        elif not isinstance(self._progress_bar_config, dict):
            raise ValueError(
                f"`self._progress_bar_config` should be of type `dict`, but is {type(self._progress_bar_config)}."
            )

        return tqdm(iterable, **self._progress_bar_config)

    def set_progress_bar_config(self, **kwargs):
        self._progress_bar_config = kwargs