pipeline_utils.py 24 KB
Newer Older
Patrick von Platen's avatar
Patrick von Platen committed
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
# coding=utf-8
# Copyright 2022 The HuggingFace Inc. team.
# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

Patrick von Platen's avatar
improve  
Patrick von Platen committed
17
import importlib
18
import inspect
Patrick von Platen's avatar
Patrick von Platen committed
19
import os
20
21
from dataclasses import dataclass
from typing import List, Optional, Union
anton-l's avatar
Style  
anton-l committed
22

23
import numpy as np
Pedro Cuenca's avatar
Pedro Cuenca committed
24
25
import torch

26
import diffusers
27
import PIL
Patrick von Platen's avatar
up  
Patrick von Platen committed
28
from huggingface_hub import snapshot_download
29
from PIL import Image
hysts's avatar
hysts committed
30
from tqdm.auto import tqdm
Patrick von Platen's avatar
Patrick von Platen committed
31

32
from . import __version__
Patrick von Platen's avatar
Patrick von Platen committed
33
from .configuration_utils import ConfigMixin
Patrick von Platen's avatar
Patrick von Platen committed
34
from .dynamic_modules_utils import get_class_from_dynamic_module
35
from .schedulers.scheduling_utils import SCHEDULER_CONFIG_NAME
36
37
38
39
40
41
42
43
44
45
46
47
48
from .utils import (
    CONFIG_NAME,
    DIFFUSERS_CACHE,
    ONNX_WEIGHTS_NAME,
    WEIGHTS_NAME,
    BaseOutput,
    is_transformers_available,
    logging,
)


if is_transformers_available():
    from transformers import PreTrainedModel
Patrick von Platen's avatar
improve  
Patrick von Platen committed
49

Patrick von Platen's avatar
Patrick von Platen committed
50

Patrick von Platen's avatar
Patrick von Platen committed
51
INDEX_FILE = "diffusion_pytorch_model.bin"
Patrick von Platen's avatar
Patrick von Platen committed
52
CUSTOM_PIPELINE_FILE_NAME = "pipeline.py"
Patrick von Platen's avatar
Patrick von Platen committed
53
54
55
56
57
58
59


logger = logging.get_logger(__name__)


LOADABLE_CLASSES = {
    "diffusers": {
Patrick von Platen's avatar
Patrick von Platen committed
60
        "ModelMixin": ["save_pretrained", "from_pretrained"],
Patrick von Platen's avatar
Patrick von Platen committed
61
        "SchedulerMixin": ["save_config", "from_config"],
Patrick von Platen's avatar
Patrick von Platen committed
62
        "DiffusionPipeline": ["save_pretrained", "from_pretrained"],
63
        "OnnxRuntimeModel": ["save_pretrained", "from_pretrained"],
Patrick von Platen's avatar
Patrick von Platen committed
64
65
    },
    "transformers": {
anton-l's avatar
anton-l committed
66
        "PreTrainedTokenizer": ["save_pretrained", "from_pretrained"],
67
        "PreTrainedTokenizerFast": ["save_pretrained", "from_pretrained"],
anton-l's avatar
anton-l committed
68
        "PreTrainedModel": ["save_pretrained", "from_pretrained"],
Suraj Patil's avatar
Suraj Patil committed
69
        "FeatureExtractionMixin": ["save_pretrained", "from_pretrained"],
Patrick von Platen's avatar
Patrick von Platen committed
70
71
72
    },
}

73
74
75
76
ALL_IMPORTABLE_CLASSES = {}
for library in LOADABLE_CLASSES:
    ALL_IMPORTABLE_CLASSES.update(LOADABLE_CLASSES[library])

Patrick von Platen's avatar
Patrick von Platen committed
77

78
79
80
81
82
83
84
85
86
87
88
89
90
91
@dataclass
class ImagePipelineOutput(BaseOutput):
    """
    Output class for image pipelines.

    Args:
        images (`List[PIL.Image.Image]` or `np.ndarray`)
            List of denoised PIL images of length `batch_size` or numpy array of shape `(batch_size, height, width,
            num_channels)`. PIL images or numpy array present the denoised images of the diffusion pipeline.
    """

    images: Union[List[PIL.Image.Image], np.ndarray]


Patrick von Platen's avatar
Patrick von Platen committed
92
class DiffusionPipeline(ConfigMixin):
93
94
95
96
97
98
99
100
101
102
103
104
    r"""
    Base class for all models.

    [`DiffusionPipeline`] takes care of storing all components (models, schedulers, processors) for diffusion pipelines
    and handles methods for loading, downloading and saving models as well as a few methods common to all pipelines to:

        - move all PyTorch modules to the device of your choice
        - enabling/disabling the progress bar for the denoising iteration

    Class attributes:

        - **config_name** ([`str`]) -- name of the config file that will store the class and module names of all
105
          components of the diffusion pipeline.
106
    """
Patrick von Platen's avatar
Patrick von Platen committed
107
108
    config_name = "model_index.json"

Patrick von Platen's avatar
up  
Patrick von Platen committed
109
    def register_modules(self, **kwargs):
110
111
        # import it here to avoid circular import
        from diffusers import pipelines
112

Patrick von Platen's avatar
Patrick von Platen committed
113
        for name, module in kwargs.items():
114
            # retrieve library
Patrick von Platen's avatar
Patrick von Platen committed
115
            library = module.__module__.split(".")[0]
116

117
118
            # check if the module is a pipeline module
            pipeline_dir = module.__module__.split(".")[-2]
Suraj Patil's avatar
Suraj Patil committed
119
120
            path = module.__module__.split(".")
            is_pipeline_module = pipeline_dir in path and hasattr(pipelines, pipeline_dir)
121

122
123
            # if library is not in LOADABLE_CLASSES, then it is a custom module.
            # Or if it's a pipeline module, then the module is inside the pipeline
124
            # folder so we set the library to module name.
125
            if library not in LOADABLE_CLASSES or is_pipeline_module:
126
                library = pipeline_dir
patil-suraj's avatar
patil-suraj committed
127

128
            # retrieve class_name
Patrick von Platen's avatar
Patrick von Platen committed
129
130
            class_name = module.__class__.__name__

131
132
            register_dict = {name: (library, class_name)}

Patrick von Platen's avatar
Patrick von Platen committed
133
            # save model index config
134
            self.register_to_config(**register_dict)
Patrick von Platen's avatar
Patrick von Platen committed
135
136
137

            # set models
            setattr(self, name, module)
138

Patrick von Platen's avatar
Patrick von Platen committed
139
    def save_pretrained(self, save_directory: Union[str, os.PathLike]):
140
141
142
143
144
145
146
147
148
        """
        Save all variables of the pipeline that can be saved and loaded as well as the pipelines configuration file to
        a directory. A pipeline variable can be saved and loaded if its class implements both a save and loading
        method. The pipeline can easily be re-loaded using the `[`~DiffusionPipeline.from_pretrained`]` class method.

        Arguments:
            save_directory (`str` or `os.PathLike`):
                Directory to which to save. Will be created if it doesn't exist.
        """
Patrick von Platen's avatar
Patrick von Platen committed
149
150
        self.save_config(save_directory)

Patrick von Platen's avatar
Patrick von Platen committed
151
        model_index_dict = dict(self.config)
Patrick von Platen's avatar
Patrick von Platen committed
152
        model_index_dict.pop("_class_name")
153
        model_index_dict.pop("_diffusers_version")
154
        model_index_dict.pop("_module", None)
Patrick von Platen's avatar
Patrick von Platen committed
155

anton-l's avatar
anton-l committed
156
157
158
        for pipeline_component_name in model_index_dict.keys():
            sub_model = getattr(self, pipeline_component_name)
            model_cls = sub_model.__class__
Patrick von Platen's avatar
Patrick von Platen committed
159
160

            save_method_name = None
anton-l's avatar
anton-l committed
161
162
163
164
165
166
167
168
169
170
171
172
173
174
            # search for the model's base class in LOADABLE_CLASSES
            for library_name, library_classes in LOADABLE_CLASSES.items():
                library = importlib.import_module(library_name)
                for base_class, save_load_methods in library_classes.items():
                    class_candidate = getattr(library, base_class)
                    if issubclass(model_cls, class_candidate):
                        # if we found a suitable base class in LOADABLE_CLASSES then grab its save method
                        save_method_name = save_load_methods[0]
                        break
                if save_method_name is not None:
                    break

            save_method = getattr(sub_model, save_method_name)
            save_method(os.path.join(save_directory, pipeline_component_name))
Patrick von Platen's avatar
Patrick von Platen committed
175

Pedro Cuenca's avatar
Pedro Cuenca committed
176
177
178
179
180
181
182
183
    def to(self, torch_device: Optional[Union[str, torch.device]] = None):
        if torch_device is None:
            return self

        module_names, _ = self.extract_init_dict(dict(self.config))
        for name in module_names.keys():
            module = getattr(self, name)
            if isinstance(module, torch.nn.Module):
184
                if module.dtype == torch.float16 and str(torch_device) in ["cpu", "mps"]:
185
186
187
188
189
190
                    logger.warning(
                        "Pipelines loaded with `torch_dtype=torch.float16` cannot run with `cpu` or `mps` device. It"
                        " is not recommended to move them to `cpu` or `mps` as running them will fail. Please make"
                        " sure to use a `cuda` device to run the pipeline in inference. due to the lack of support for"
                        " `float16` operations on those devices in PyTorch. Please remove the"
                        " `torch_dtype=torch.float16` argument, or use a `cuda` device to run inference."
191
                    )
Pedro Cuenca's avatar
Pedro Cuenca committed
192
193
194
195
196
                module.to(torch_device)
        return self

    @property
    def device(self) -> torch.device:
197
198
199
200
        r"""
        Returns:
            `torch.device`: The torch device on which the pipeline is located.
        """
Pedro Cuenca's avatar
Pedro Cuenca committed
201
202
203
204
205
206
207
        module_names, _ = self.extract_init_dict(dict(self.config))
        for name in module_names.keys():
            module = getattr(self, name)
            if isinstance(module, torch.nn.Module):
                return module.device
        return torch.device("cpu")

Patrick von Platen's avatar
Patrick von Platen committed
208
209
    @classmethod
    def from_pretrained(cls, pretrained_model_name_or_path: Optional[Union[str, os.PathLike]], **kwargs):
210
        r"""
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
        Instantiate a PyTorch diffusion pipeline from pre-trained pipeline weights.

        The pipeline is set in evaluation mode by default using `model.eval()` (Dropout modules are deactivated).

        The warning *Weights from XXX not initialized from pretrained model* means that the weights of XXX do not come
        pretrained with the rest of the model. It is up to you to train those weights with a downstream fine-tuning
        task.

        The warning *Weights from XXX not used in YYY* means that the layer XXX is not used by YYY, therefore those
        weights are discarded.

        Parameters:
            pretrained_model_name_or_path (`str` or `os.PathLike`, *optional*):
                Can be either:

                    - A string, the *repo id* of a pretrained pipeline hosted inside a model repo on
                      https://huggingface.co/ Valid repo ids have to be located under a user or organization name, like
                      `CompVis/ldm-text2im-large-256`.
                    - A path to a *directory* containing pipeline weights saved using
                      [`~DiffusionPipeline.save_pretrained`], e.g., `./my_pipeline_directory/`.
            torch_dtype (`str` or `torch.dtype`, *optional*):
                Override the default `torch.dtype` and load the model under this dtype. If `"auto"` is passed the dtype
                will be automatically derived from the model's weights.
Patrick von Platen's avatar
Patrick von Platen committed
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
            custom_pipeline (`str`, *optional*):

                <Tip warning={true}>

                    This is an experimental feature and is likely to change in the future.

                </Tip>

                Can be either:

                    - A string, the *repo id* of a custom pipeline hosted inside a model repo on
                      https://huggingface.co/. Valid repo ids have to be located under a user or organization name,
                      like `hf-internal-testing/diffusers-dummy-pipeline`.

                        <Tip>

                         It is required that the model repo has a file, called `pipeline.py` that defines the custom
                         pipeline.

                        </Tip>

                    - A string, the *file name* of a community pipeline hosted on GitHub under
                      https://github.com/huggingface/diffusers/tree/main/examples/community. Valid file names have to
                      match exactly the file name without `.py` located under the above link, *e.g.*
                      `clip_guided_stable_diffusion`.

                        <Tip>

                         Community pipelines are always loaded from the current `main` branch of GitHub.

                        </Tip>

                    - A path to a *directory* containing a custom pipeline, e.g., `./my_pipeline_directory/`.

                        <Tip>

                         It is required that the directory has a file, called `pipeline.py` that defines the custom
                         pipeline.

                        </Tip>

                For more information on how to load and create custom pipelines, please have a look at [Loading and
                Creating Custom
                Pipelines](https://huggingface.co/docs/diffusers/main/en/using-diffusers/custom_pipelines)

            torch_dtype (`str` or `torch.dtype`, *optional*):
280
281
282
283
284
285
286
287
288
289
            force_download (`bool`, *optional*, defaults to `False`):
                Whether or not to force the (re-)download of the model weights and configuration files, overriding the
                cached versions if they exist.
            resume_download (`bool`, *optional*, defaults to `False`):
                Whether or not to delete incompletely received files. Will attempt to resume the download if such a
                file exists.
            proxies (`Dict[str, str]`, *optional*):
                A dictionary of proxy servers to use by protocol or endpoint, e.g., `{'http': 'foo.bar:3128',
                'http://hostname': 'foo.bar:4012'}`. The proxies are used on each request.
            output_loading_info(`bool`, *optional*, defaults to `False`):
290
                Whether or not to also return a dictionary containing missing keys, unexpected keys and error messages.
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
            local_files_only(`bool`, *optional*, defaults to `False`):
                Whether or not to only look at local files (i.e., do not try to download the model).
            use_auth_token (`str` or *bool*, *optional*):
                The token to use as HTTP bearer authorization for remote files. If `True`, will use the token generated
                when running `huggingface-cli login` (stored in `~/.huggingface`).
            revision (`str`, *optional*, defaults to `"main"`):
                The specific model version to use. It can be a branch name, a tag name, or a commit id, since we use a
                git-based system for storing models and other artifacts on huggingface.co, so `revision` can be any
                identifier allowed by git.
            mirror (`str`, *optional*):
                Mirror source to accelerate downloads in China. If you are from China and have an accessibility
                problem, you can set this option to resolve it. Note that we do not guarantee the timeliness or safety.
                Please refer to the mirror site for more information. specify the folder name here.

            kwargs (remaining dictionary of keyword arguments, *optional*):
                Can be used to overwrite load - and saveable variables - *i.e.* the pipeline components - of the
307
308
                specific pipeline class. The overwritten components are then directly passed to the pipelines
                `__init__` method. See example below for more information.
309
310
311

        <Tip>

312
313
         It is required to be logged in (`huggingface-cli login`) when you want to use private or [gated
         models](https://huggingface.co/docs/hub/models-gated#gated-models), *e.g.* `"CompVis/stable-diffusion-v1-4"`
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334

        </Tip>

        <Tip>

        Activate the special ["offline-mode"](https://huggingface.co/diffusers/installation.html#offline-mode) to use
        this method in a firewalled environment.

        </Tip>

        Examples:

        ```py
        >>> from diffusers import DiffusionPipeline

        >>> # Download pipeline from huggingface.co and cache.
        >>> pipeline = DiffusionPipeline.from_pretrained("CompVis/ldm-text2im-large-256")

        >>> # Download pipeline that requires an authorization token
        >>> # For more information on access tokens, please refer to this section
        >>> # of the documentation](https://huggingface.co/docs/hub/security-tokens)
335
        >>> pipeline = DiffusionPipeline.from_pretrained("CompVis/stable-diffusion-v1-4")
336
337
338
339
340

        >>> # Download pipeline, but overwrite scheduler
        >>> from diffusers import LMSDiscreteScheduler

        >>> scheduler = LMSDiscreteScheduler(beta_start=0.00085, beta_end=0.012, beta_schedule="scaled_linear")
341
        >>> pipeline = DiffusionPipeline.from_pretrained("CompVis/stable-diffusion-v1-4", scheduler=scheduler)
342
        ```
343
344
345
346
347
348
        """
        cache_dir = kwargs.pop("cache_dir", DIFFUSERS_CACHE)
        resume_download = kwargs.pop("resume_download", False)
        proxies = kwargs.pop("proxies", None)
        local_files_only = kwargs.pop("local_files_only", False)
        use_auth_token = kwargs.pop("use_auth_token", None)
349
        revision = kwargs.pop("revision", None)
350
        torch_dtype = kwargs.pop("torch_dtype", None)
Patrick von Platen's avatar
Patrick von Platen committed
351
        custom_pipeline = kwargs.pop("custom_pipeline", None)
352
        provider = kwargs.pop("provider", None)
353
        sess_options = kwargs.pop("sess_options", None)
354
        device_map = kwargs.pop("device_map", None)
Patrick von Platen's avatar
Patrick von Platen committed
355

patil-suraj's avatar
patil-suraj committed
356
        # 1. Download the checkpoints and configs
Patrick von Platen's avatar
Patrick von Platen committed
357
        # use snapshot download here to get it working from from_pretrained
Patrick von Platen's avatar
Patrick von Platen committed
358
        if not os.path.isdir(pretrained_model_name_or_path):
359
360
361
362
363
364
365
366
367
368
369
370
371
372
            config_dict = cls.get_config_dict(
                pretrained_model_name_or_path,
                cache_dir=cache_dir,
                resume_download=resume_download,
                proxies=proxies,
                local_files_only=local_files_only,
                use_auth_token=use_auth_token,
                revision=revision,
            )
            # make sure we only download sub-folders and `diffusers` filenames
            folder_names = [k for k in config_dict.keys() if not k.startswith("_")]
            allow_patterns = [os.path.join(k, "*") for k in folder_names]
            allow_patterns += [WEIGHTS_NAME, SCHEDULER_CONFIG_NAME, CONFIG_NAME, ONNX_WEIGHTS_NAME, cls.config_name]

Patrick von Platen's avatar
Patrick von Platen committed
373
374
375
            if custom_pipeline is not None:
                allow_patterns += [CUSTOM_PIPELINE_FILE_NAME]

376
377
378
379
            user_agent = {"diffusers": __version__, "pipeline_class": config_dict["_class_name"]}
            if custom_pipeline is not None:
                user_agent["custom_pipeline"] = custom_pipeline

380
            # download all allow_patterns
381
382
383
384
385
386
387
            cached_folder = snapshot_download(
                pretrained_model_name_or_path,
                cache_dir=cache_dir,
                resume_download=resume_download,
                proxies=proxies,
                local_files_only=local_files_only,
                use_auth_token=use_auth_token,
388
                revision=revision,
389
                allow_patterns=allow_patterns,
390
                user_agent=user_agent,
391
            )
Patrick von Platen's avatar
Patrick von Platen committed
392
393
        else:
            cached_folder = pretrained_model_name_or_path
394

patil-suraj's avatar
patil-suraj committed
395
        config_dict = cls.get_config_dict(cached_folder)
396

Patrick von Platen's avatar
Patrick von Platen committed
397
        # 2. Load the pipeline class, if using custom module then load it from the hub
398
        # if we load from explicit class, let's use it
Patrick von Platen's avatar
Patrick von Platen committed
399
400
401
402
403
        if custom_pipeline is not None:
            pipeline_class = get_class_from_dynamic_module(
                custom_pipeline, module_file=CUSTOM_PIPELINE_FILE_NAME, cache_dir=custom_pipeline
            )
        elif cls != DiffusionPipeline:
404
405
            pipeline_class = cls
        else:
Patrick von Platen's avatar
Patrick von Platen committed
406
407
408
            diffusers_module = importlib.import_module(cls.__module__.split(".")[0])
            pipeline_class = getattr(diffusers_module, config_dict["_class_name"])

409
410
411
        # some modules can be passed directly to the init
        # in this case they are already instantiated in `kwargs`
        # extract them here
Patrick von Platen's avatar
Patrick von Platen committed
412
        expected_modules = set(inspect.signature(pipeline_class.__init__).parameters.keys()) - set(["self"])
413
414
        passed_class_obj = {k: kwargs.pop(k) for k in expected_modules if k in kwargs}

415
        init_dict, _ = pipeline_class.extract_init_dict(config_dict, **kwargs)
Patrick von Platen's avatar
Patrick von Platen committed
416
417

        init_kwargs = {}
418

419
420
        # import it here to avoid circular import
        from diffusers import pipelines
421

Patrick von Platen's avatar
Patrick von Platen committed
422
        # 3. Load each module in the pipeline
patil-suraj's avatar
patil-suraj committed
423
        for name, (library_name, class_name) in init_dict.items():
424
425
426
427
            # 3.1 - now that JAX/Flax is an official framework of the library, we might load from Flax names
            if class_name.startswith("Flax"):
                class_name = class_name[4:]

428
            is_pipeline_module = hasattr(pipelines, library_name)
429
430
            loaded_sub_model = None

431
            # if the model is in a pipeline module, then we load it from the pipeline
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
            if name in passed_class_obj:
                # 1. check that passed_class_obj has correct parent class
                if not is_pipeline_module:
                    library = importlib.import_module(library_name)
                    class_obj = getattr(library, class_name)
                    importable_classes = LOADABLE_CLASSES[library_name]
                    class_candidates = {c: getattr(library, c) for c in importable_classes.keys()}

                    expected_class_obj = None
                    for class_name, class_candidate in class_candidates.items():
                        if issubclass(class_obj, class_candidate):
                            expected_class_obj = class_candidate

                    if not issubclass(passed_class_obj[name].__class__, expected_class_obj):
                        raise ValueError(
                            f"{passed_class_obj[name]} is of type: {type(passed_class_obj[name])}, but should be"
                            f" {expected_class_obj}"
                        )
                else:
                    logger.warn(
                        f"You have passed a non-standard module {passed_class_obj[name]}. We cannot verify whether it"
                        " has the correct type"
                    )

                # set passed class object
                loaded_sub_model = passed_class_obj[name]
            elif is_pipeline_module:
459
460
461
                pipeline_module = getattr(pipelines, library_name)
                class_obj = getattr(pipeline_module, class_name)
                importable_classes = ALL_IMPORTABLE_CLASSES
Patrick von Platen's avatar
Patrick von Platen committed
462
                class_candidates = {c: class_obj for c in importable_classes.keys()}
patil-suraj's avatar
patil-suraj committed
463
            else:
patil-suraj's avatar
patil-suraj committed
464
                # else we just import it from the library.
patil-suraj's avatar
patil-suraj committed
465
466
                library = importlib.import_module(library_name)
                class_obj = getattr(library, class_name)
467
                importable_classes = LOADABLE_CLASSES[library_name]
patil-suraj's avatar
patil-suraj committed
468
                class_candidates = {c: getattr(library, c) for c in importable_classes.keys()}
469

470
471
472
473
474
            if loaded_sub_model is None:
                load_method_name = None
                for class_name, class_candidate in class_candidates.items():
                    if issubclass(class_obj, class_candidate):
                        load_method_name = importable_classes[class_name][1]
Patrick von Platen's avatar
Patrick von Platen committed
475

476
                load_method = getattr(class_obj, load_method_name)
Patrick von Platen's avatar
Patrick von Platen committed
477

478
479
480
                loading_kwargs = {}
                if issubclass(class_obj, torch.nn.Module):
                    loading_kwargs["torch_dtype"] = torch_dtype
481
482
                if issubclass(class_obj, diffusers.OnnxRuntimeModel):
                    loading_kwargs["provider"] = provider
483
                    loading_kwargs["sess_options"] = sess_options
484

485
486
487
488
489
490
491
                if (
                    issubclass(class_obj, diffusers.ModelMixin)
                    or is_transformers_available()
                    and issubclass(class_obj, PreTrainedModel)
                ):
                    loading_kwargs["device_map"] = device_map

492
493
                # check if the module is in a subdirectory
                if os.path.isdir(os.path.join(cached_folder, name)):
494
                    loaded_sub_model = load_method(os.path.join(cached_folder, name), **loading_kwargs)
495
496
                else:
                    # else load from the root directory
497
                    loaded_sub_model = load_method(cached_folder, **loading_kwargs)
Patrick von Platen's avatar
Patrick von Platen committed
498

499
            init_kwargs[name] = loaded_sub_model  # UNet(...), # DiffusionSchedule(...)
Patrick von Platen's avatar
Patrick von Platen committed
500

Patrick von Platen's avatar
Patrick von Platen committed
501
502
503
504
505
506
        # 4. Potentially add passed objects if expected
        missing_modules = set(expected_modules) - set(init_kwargs.keys())
        if len(missing_modules) > 0 and missing_modules <= set(passed_class_obj.keys()):
            for module in missing_modules:
                init_kwargs[module] = passed_class_obj[module]
        elif len(missing_modules) > 0:
507
            passed_modules = set(list(init_kwargs.keys()) + list(passed_class_obj.keys()))
Patrick von Platen's avatar
Patrick von Platen committed
508
509
510
511
512
            raise ValueError(
                f"Pipeline {pipeline_class} expected {expected_modules}, but only {passed_modules} were passed."
            )

        # 5. Instantiate the pipeline
513
        model = pipeline_class(**init_kwargs)
Patrick von Platen's avatar
Patrick von Platen committed
514
        return model
515
516
517
518
519
520
521
522
523
524
525
526

    @staticmethod
    def numpy_to_pil(images):
        """
        Convert a numpy image or a batch of images to a PIL image.
        """
        if images.ndim == 3:
            images = images[None, ...]
        images = (images * 255).round().astype("uint8")
        pil_images = [Image.fromarray(image) for image in images]

        return pil_images
hysts's avatar
hysts committed
527
528
529
530
531
532
533
534
535
536
537
538
539

    def progress_bar(self, iterable):
        if not hasattr(self, "_progress_bar_config"):
            self._progress_bar_config = {}
        elif not isinstance(self._progress_bar_config, dict):
            raise ValueError(
                f"`self._progress_bar_config` should be of type `dict`, but is {type(self._progress_bar_config)}."
            )

        return tqdm(iterable, **self._progress_bar_config)

    def set_progress_bar_config(self, **kwargs):
        self._progress_bar_config = kwargs