test_pipelines.py 46.8 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
# coding=utf-8
# Copyright 2022 HuggingFace Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

16
import gc
17
import random
18
19
20
21
22
23
24
import tempfile
import unittest

import numpy as np
import torch

import PIL
25
from datasets import load_dataset
26
from diffusers import (
27
    AutoencoderKL,
28
29
30
31
32
33
34
35
36
37
38
39
40
    DDIMPipeline,
    DDIMScheduler,
    DDPMPipeline,
    DDPMScheduler,
    KarrasVePipeline,
    KarrasVeScheduler,
    LDMPipeline,
    LDMTextToImagePipeline,
    LMSDiscreteScheduler,
    PNDMPipeline,
    PNDMScheduler,
    ScoreSdeVePipeline,
    ScoreSdeVeScheduler,
41
42
    StableDiffusionImg2ImgPipeline,
    StableDiffusionInpaintPipeline,
43
    StableDiffusionPipeline,
44
    UNet2DConditionModel,
45
    UNet2DModel,
46
    VQModel,
47
48
)
from diffusers.pipeline_utils import DiffusionPipeline
49
50
51
from diffusers.testing_utils import floats_tensor, slow, torch_device
from PIL import Image
from transformers import CLIPTextConfig, CLIPTextModel, CLIPTokenizer
52
53
54
55
56


torch.backends.cuda.matmul.allow_tf32 = False


hysts's avatar
hysts committed
57
58
59
60
61
62
63
64
65
66
67
68
69
def test_progress_bar(capsys):
    model = UNet2DModel(
        block_out_channels=(32, 64),
        layers_per_block=2,
        sample_size=32,
        in_channels=3,
        out_channels=3,
        down_block_types=("DownBlock2D", "AttnDownBlock2D"),
        up_block_types=("AttnUpBlock2D", "UpBlock2D"),
    )
    scheduler = DDPMScheduler(num_train_timesteps=10)

    ddpm = DDPMPipeline(model, scheduler).to(torch_device)
70
    ddpm(output_type="numpy").images
hysts's avatar
hysts committed
71
72
73
74
    captured = capsys.readouterr()
    assert "10/10" in captured.err, "Progress bar has to be displayed"

    ddpm.set_progress_bar_config(disable=True)
75
    ddpm(output_type="numpy").images
hysts's avatar
hysts committed
76
77
78
79
    captured = capsys.readouterr()
    assert captured.err == "", "Progress bar should be disabled"


80
class PipelineFastTests(unittest.TestCase):
81
82
83
84
85
86
    def tearDown(self):
        # clean up the VRAM after each test
        super().tearDown()
        gc.collect()
        torch.cuda.empty_cache()

87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
    @property
    def dummy_image(self):
        batch_size = 1
        num_channels = 3
        sizes = (32, 32)

        image = floats_tensor((batch_size, num_channels) + sizes, rng=random.Random(0)).to(torch_device)
        return image

    @property
    def dummy_uncond_unet(self):
        torch.manual_seed(0)
        model = UNet2DModel(
            block_out_channels=(32, 64),
            layers_per_block=2,
            sample_size=32,
            in_channels=3,
            out_channels=3,
            down_block_types=("DownBlock2D", "AttnDownBlock2D"),
            up_block_types=("AttnUpBlock2D", "UpBlock2D"),
        )
        return model

    @property
    def dummy_cond_unet(self):
        torch.manual_seed(0)
        model = UNet2DConditionModel(
            block_out_channels=(32, 64),
            layers_per_block=2,
            sample_size=32,
            in_channels=4,
            out_channels=4,
            down_block_types=("DownBlock2D", "CrossAttnDownBlock2D"),
            up_block_types=("CrossAttnUpBlock2D", "UpBlock2D"),
            cross_attention_dim=32,
        )
        return model

    @property
    def dummy_vq_model(self):
        torch.manual_seed(0)
        model = VQModel(
            block_out_channels=[32, 64],
            in_channels=3,
            out_channels=3,
            down_block_types=["DownEncoderBlock2D", "DownEncoderBlock2D"],
            up_block_types=["UpDecoderBlock2D", "UpDecoderBlock2D"],
            latent_channels=3,
        )
        return model

    @property
    def dummy_vae(self):
        torch.manual_seed(0)
        model = AutoencoderKL(
            block_out_channels=[32, 64],
            in_channels=3,
            out_channels=3,
            down_block_types=["DownEncoderBlock2D", "DownEncoderBlock2D"],
            up_block_types=["UpDecoderBlock2D", "UpDecoderBlock2D"],
            latent_channels=4,
        )
        return model

    @property
    def dummy_text_encoder(self):
        torch.manual_seed(0)
        config = CLIPTextConfig(
            bos_token_id=0,
            eos_token_id=2,
            hidden_size=32,
            intermediate_size=37,
            layer_norm_eps=1e-05,
            num_attention_heads=4,
            num_hidden_layers=5,
            pad_token_id=1,
            vocab_size=1000,
        )
        return CLIPTextModel(config)

    @property
    def dummy_safety_checker(self):
        def check(images, *args, **kwargs):
            return images, False

        return check

    @property
    def dummy_extractor(self):
        def extract(*args, **kwargs):
            class Out:
                def __init__(self):
                    self.pixel_values = torch.ones([0])

                def to(self, device):
                    self.pixel_values.to(device)
                    return self

            return Out()

        return extract

    def test_ddim(self):
        unet = self.dummy_uncond_unet
        scheduler = DDIMScheduler(tensor_format="pt")

        ddpm = DDIMPipeline(unet=unet, scheduler=scheduler)
        ddpm.to(torch_device)
195
        ddpm.set_progress_bar_config(disable=None)
196

197
198
199
200
        # Warmup pass when using mps (see #372)
        if torch_device == "mps":
            _ = ddpm(num_inference_steps=1)

201
        generator = torch.manual_seed(0)
202
203
204
205
        image = ddpm(generator=generator, num_inference_steps=2, output_type="numpy").images

        generator = torch.manual_seed(0)
        image_from_tuple = ddpm(generator=generator, num_inference_steps=2, output_type="numpy", return_dict=False)[0]
206
207

        image_slice = image[0, -3:, -3:, -1]
208
        image_from_tuple_slice = image_from_tuple[0, -3:, -3:, -1]
209
210
211
212
213

        assert image.shape == (1, 32, 32, 3)
        expected_slice = np.array(
            [1.000e00, 5.717e-01, 4.717e-01, 1.000e00, 0.000e00, 1.000e00, 3.000e-04, 0.000e00, 9.000e-04]
        )
214
215
216
        tolerance = 1e-2 if torch_device != "mps" else 3e-2
        assert np.abs(image_slice.flatten() - expected_slice).max() < tolerance
        assert np.abs(image_from_tuple_slice.flatten() - expected_slice).max() < tolerance
217
218
219
220
221
222
223

    def test_pndm_cifar10(self):
        unet = self.dummy_uncond_unet
        scheduler = PNDMScheduler(tensor_format="pt")

        pndm = PNDMPipeline(unet=unet, scheduler=scheduler)
        pndm.to(torch_device)
224
        pndm.set_progress_bar_config(disable=None)
225
226
227
228

        generator = torch.manual_seed(0)
        image = pndm(generator=generator, num_inference_steps=20, output_type="numpy").images

229
        generator = torch.manual_seed(0)
230
        image_from_tuple = pndm(generator=generator, num_inference_steps=20, output_type="numpy", return_dict=False)[0]
231
232

        image_slice = image[0, -3:, -3:, -1]
233
        image_from_tuple_slice = image_from_tuple[0, -3:, -3:, -1]
234
235
236
237

        assert image.shape == (1, 32, 32, 3)
        expected_slice = np.array([1.0, 1.0, 0.0, 1.0, 0.0, 1.0, 0.0, 0.0, 0.0])
        assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-2
238
        assert np.abs(image_from_tuple_slice.flatten() - expected_slice).max() < 1e-2
239
240
241
242
243
244
245
246
247
248

    def test_ldm_text2img(self):
        unet = self.dummy_cond_unet
        scheduler = DDIMScheduler(tensor_format="pt")
        vae = self.dummy_vae
        bert = self.dummy_text_encoder
        tokenizer = CLIPTokenizer.from_pretrained("hf-internal-testing/tiny-random-clip")

        ldm = LDMTextToImagePipeline(vqvae=vae, bert=bert, tokenizer=tokenizer, unet=unet, scheduler=scheduler)
        ldm.to(torch_device)
249
        ldm.set_progress_bar_config(disable=None)
250
251

        prompt = "A painting of a squirrel eating a burger"
252
253
254
255
256
257
258
259

        # Warmup pass when using mps (see #372)
        if torch_device == "mps":
            generator = torch.manual_seed(0)
            _ = ldm([prompt], generator=generator, guidance_scale=6.0, num_inference_steps=1, output_type="numpy")[
                "sample"
            ]

260
261
262
263
264
        generator = torch.manual_seed(0)
        image = ldm([prompt], generator=generator, guidance_scale=6.0, num_inference_steps=2, output_type="numpy")[
            "sample"
        ]

265
266
267
268
269
270
271
272
273
274
        generator = torch.manual_seed(0)
        image_from_tuple = ldm(
            [prompt],
            generator=generator,
            guidance_scale=6.0,
            num_inference_steps=2,
            output_type="numpy",
            return_dict=False,
        )[0]

275
        image_slice = image[0, -3:, -3:, -1]
276
        image_from_tuple_slice = image_from_tuple[0, -3:, -3:, -1]
277
278
279
280

        assert image.shape == (1, 64, 64, 3)
        expected_slice = np.array([0.5074, 0.5026, 0.4998, 0.4056, 0.3523, 0.4649, 0.5289, 0.5299, 0.4897])
        assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-2
281
        assert np.abs(image_from_tuple_slice.flatten() - expected_slice).max() < 1e-2
282
283

    def test_stable_diffusion_ddim(self):
284
        device = "cpu"  # ensure determinism for the device-dependent torch.Generator
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
        unet = self.dummy_cond_unet
        scheduler = DDIMScheduler(
            beta_start=0.00085,
            beta_end=0.012,
            beta_schedule="scaled_linear",
            clip_sample=False,
            set_alpha_to_one=False,
        )

        vae = self.dummy_vae
        bert = self.dummy_text_encoder
        tokenizer = CLIPTokenizer.from_pretrained("hf-internal-testing/tiny-random-clip")

        # make sure here that pndm scheduler skips prk
        sd_pipe = StableDiffusionPipeline(
            unet=unet,
            scheduler=scheduler,
            vae=vae,
            text_encoder=bert,
            tokenizer=tokenizer,
            safety_checker=self.dummy_safety_checker,
            feature_extractor=self.dummy_extractor,
        )
308
        sd_pipe = sd_pipe.to(device)
309
        sd_pipe.set_progress_bar_config(disable=None)
310
311

        prompt = "A painting of a squirrel eating a burger"
312

313
314
        generator = torch.Generator(device=device).manual_seed(0)
        output = sd_pipe([prompt], generator=generator, guidance_scale=6.0, num_inference_steps=2, output_type="np")
315
        image = output.images
316

317
318
319
320
321
322
323
324
325
        generator = torch.Generator(device=device).manual_seed(0)
        image_from_tuple = sd_pipe(
            [prompt],
            generator=generator,
            guidance_scale=6.0,
            num_inference_steps=2,
            output_type="np",
            return_dict=False,
        )[0]
326
327

        image_slice = image[0, -3:, -3:, -1]
328
        image_from_tuple_slice = image_from_tuple[0, -3:, -3:, -1]
329
330
331
332

        assert image.shape == (1, 128, 128, 3)
        expected_slice = np.array([0.5112, 0.4692, 0.4715, 0.5206, 0.4894, 0.5114, 0.5096, 0.4932, 0.4755])
        assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-2
333
        assert np.abs(image_from_tuple_slice.flatten() - expected_slice).max() < 1e-2
334
335

    def test_stable_diffusion_pndm(self):
336
        device = "cpu"  # ensure determinism for the device-dependent torch.Generator
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
        unet = self.dummy_cond_unet
        scheduler = PNDMScheduler(tensor_format="pt", skip_prk_steps=True)
        vae = self.dummy_vae
        bert = self.dummy_text_encoder
        tokenizer = CLIPTokenizer.from_pretrained("hf-internal-testing/tiny-random-clip")

        # make sure here that pndm scheduler skips prk
        sd_pipe = StableDiffusionPipeline(
            unet=unet,
            scheduler=scheduler,
            vae=vae,
            text_encoder=bert,
            tokenizer=tokenizer,
            safety_checker=self.dummy_safety_checker,
            feature_extractor=self.dummy_extractor,
        )
353
        sd_pipe = sd_pipe.to(device)
354
        sd_pipe.set_progress_bar_config(disable=None)
355
356

        prompt = "A painting of a squirrel eating a burger"
357
358
        generator = torch.Generator(device=device).manual_seed(0)
        output = sd_pipe([prompt], generator=generator, guidance_scale=6.0, num_inference_steps=2, output_type="np")
359

360
361
362
363
364
365
366
367
368
369
370
        image = output.images

        generator = torch.Generator(device=device).manual_seed(0)
        image_from_tuple = sd_pipe(
            [prompt],
            generator=generator,
            guidance_scale=6.0,
            num_inference_steps=2,
            output_type="np",
            return_dict=False,
        )[0]
371
372

        image_slice = image[0, -3:, -3:, -1]
373
        image_from_tuple_slice = image_from_tuple[0, -3:, -3:, -1]
374
375
376
377

        assert image.shape == (1, 128, 128, 3)
        expected_slice = np.array([0.4937, 0.4649, 0.4716, 0.5145, 0.4889, 0.513, 0.513, 0.4905, 0.4738])
        assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-2
378
        assert np.abs(image_from_tuple_slice.flatten() - expected_slice).max() < 1e-2
379
380

    def test_stable_diffusion_k_lms(self):
381
        device = "cpu"  # ensure determinism for the device-dependent torch.Generator
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
        unet = self.dummy_cond_unet
        scheduler = LMSDiscreteScheduler(beta_start=0.00085, beta_end=0.012, beta_schedule="scaled_linear")
        vae = self.dummy_vae
        bert = self.dummy_text_encoder
        tokenizer = CLIPTokenizer.from_pretrained("hf-internal-testing/tiny-random-clip")

        # make sure here that pndm scheduler skips prk
        sd_pipe = StableDiffusionPipeline(
            unet=unet,
            scheduler=scheduler,
            vae=vae,
            text_encoder=bert,
            tokenizer=tokenizer,
            safety_checker=self.dummy_safety_checker,
            feature_extractor=self.dummy_extractor,
        )
398
        sd_pipe = sd_pipe.to(device)
399
        sd_pipe.set_progress_bar_config(disable=None)
400
401

        prompt = "A painting of a squirrel eating a burger"
402
403
        generator = torch.Generator(device=device).manual_seed(0)
        output = sd_pipe([prompt], generator=generator, guidance_scale=6.0, num_inference_steps=2, output_type="np")
404

405
406
407
408
409
410
411
412
413
414
415
        image = output.images

        generator = torch.Generator(device=device).manual_seed(0)
        image_from_tuple = sd_pipe(
            [prompt],
            generator=generator,
            guidance_scale=6.0,
            num_inference_steps=2,
            output_type="np",
            return_dict=False,
        )[0]
416
417

        image_slice = image[0, -3:, -3:, -1]
418
        image_from_tuple_slice = image_from_tuple[0, -3:, -3:, -1]
419
420
421
422

        assert image.shape == (1, 128, 128, 3)
        expected_slice = np.array([0.5067, 0.4689, 0.4614, 0.5233, 0.4903, 0.5112, 0.524, 0.5069, 0.4785])
        assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-2
423
        assert np.abs(image_from_tuple_slice.flatten() - expected_slice).max() < 1e-2
424

425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
    def test_stable_diffusion_attention_chunk(self):
        device = "cpu"  # ensure determinism for the device-dependent torch.Generator
        unet = self.dummy_cond_unet
        scheduler = LMSDiscreteScheduler(beta_start=0.00085, beta_end=0.012, beta_schedule="scaled_linear")
        vae = self.dummy_vae
        bert = self.dummy_text_encoder
        tokenizer = CLIPTokenizer.from_pretrained("hf-internal-testing/tiny-random-clip")

        # make sure here that pndm scheduler skips prk
        sd_pipe = StableDiffusionPipeline(
            unet=unet,
            scheduler=scheduler,
            vae=vae,
            text_encoder=bert,
            tokenizer=tokenizer,
            safety_checker=self.dummy_safety_checker,
            feature_extractor=self.dummy_extractor,
        )
        sd_pipe = sd_pipe.to(device)
        sd_pipe.set_progress_bar_config(disable=None)

        prompt = "A painting of a squirrel eating a burger"
        generator = torch.Generator(device=device).manual_seed(0)
        output_1 = sd_pipe([prompt], generator=generator, guidance_scale=6.0, num_inference_steps=2, output_type="np")

        # make sure chunking the attention yields the same result
        sd_pipe.enable_attention_slicing(slice_size=1)
        generator = torch.Generator(device=device).manual_seed(0)
        output_2 = sd_pipe([prompt], generator=generator, guidance_scale=6.0, num_inference_steps=2, output_type="np")

        assert np.abs(output_2.images.flatten() - output_1.images.flatten()).max() < 1e-4

457
458
459
460
461
462
    def test_score_sde_ve_pipeline(self):
        unet = self.dummy_uncond_unet
        scheduler = ScoreSdeVeScheduler(tensor_format="pt")

        sde_ve = ScoreSdeVePipeline(unet=unet, scheduler=scheduler)
        sde_ve.to(torch_device)
463
        sde_ve.set_progress_bar_config(disable=None)
464
465

        torch.manual_seed(0)
466
467
468
469
        image = sde_ve(num_inference_steps=2, output_type="numpy").images

        torch.manual_seed(0)
        image_from_tuple = sde_ve(num_inference_steps=2, output_type="numpy", return_dict=False)[0]
470
471

        image_slice = image[0, -3:, -3:, -1]
472
        image_from_tuple_slice = image_from_tuple[0, -3:, -3:, -1]
473
474
475
476
477

        assert image.shape == (1, 32, 32, 3)

        expected_slice = np.array([0.0, 1.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0])
        assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-2
478
        assert np.abs(image_from_tuple_slice.flatten() - expected_slice).max() < 1e-2
479
480
481
482
483
484
485
486

    def test_ldm_uncond(self):
        unet = self.dummy_uncond_unet
        scheduler = DDIMScheduler(tensor_format="pt")
        vae = self.dummy_vq_model

        ldm = LDMPipeline(unet=unet, vqvae=vae, scheduler=scheduler)
        ldm.to(torch_device)
487
        ldm.set_progress_bar_config(disable=None)
488

489
490
491
492
493
        # Warmup pass when using mps (see #372)
        if torch_device == "mps":
            generator = torch.manual_seed(0)
            _ = ldm(generator=generator, num_inference_steps=1, output_type="numpy").images

494
        generator = torch.manual_seed(0)
495
496
497
498
        image = ldm(generator=generator, num_inference_steps=2, output_type="numpy").images

        generator = torch.manual_seed(0)
        image_from_tuple = ldm(generator=generator, num_inference_steps=2, output_type="numpy", return_dict=False)[0]
499
500

        image_slice = image[0, -3:, -3:, -1]
501
        image_from_tuple_slice = image_from_tuple[0, -3:, -3:, -1]
502
503
504
505

        assert image.shape == (1, 64, 64, 3)
        expected_slice = np.array([0.8512, 0.818, 0.6411, 0.6808, 0.4465, 0.5618, 0.46, 0.6231, 0.5172])
        assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-2
506
        assert np.abs(image_from_tuple_slice.flatten() - expected_slice).max() < 1e-2
507
508
509
510
511
512
513

    def test_karras_ve_pipeline(self):
        unet = self.dummy_uncond_unet
        scheduler = KarrasVeScheduler(tensor_format="pt")

        pipe = KarrasVePipeline(unet=unet, scheduler=scheduler)
        pipe.to(torch_device)
514
        pipe.set_progress_bar_config(disable=None)
515
516

        generator = torch.manual_seed(0)
517
518
519
520
        image = pipe(num_inference_steps=2, generator=generator, output_type="numpy").images

        generator = torch.manual_seed(0)
        image_from_tuple = pipe(num_inference_steps=2, generator=generator, output_type="numpy", return_dict=False)[0]
521
522

        image_slice = image[0, -3:, -3:, -1]
523
524
        image_from_tuple_slice = image_from_tuple[0, -3:, -3:, -1]

525
526
527
        assert image.shape == (1, 32, 32, 3)
        expected_slice = np.array([0.0, 1.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0])
        assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-2
528
        assert np.abs(image_from_tuple_slice.flatten() - expected_slice).max() < 1e-2
529
530

    def test_stable_diffusion_img2img(self):
531
        device = "cpu"  # ensure determinism for the device-dependent torch.Generator
532
533
534
535
536
537
        unet = self.dummy_cond_unet
        scheduler = PNDMScheduler(tensor_format="pt", skip_prk_steps=True)
        vae = self.dummy_vae
        bert = self.dummy_text_encoder
        tokenizer = CLIPTokenizer.from_pretrained("hf-internal-testing/tiny-random-clip")

538
        init_image = self.dummy_image.to(device)
539
540
541
542
543
544
545
546
547
548
549

        # make sure here that pndm scheduler skips prk
        sd_pipe = StableDiffusionImg2ImgPipeline(
            unet=unet,
            scheduler=scheduler,
            vae=vae,
            text_encoder=bert,
            tokenizer=tokenizer,
            safety_checker=self.dummy_safety_checker,
            feature_extractor=self.dummy_extractor,
        )
550
        sd_pipe = sd_pipe.to(device)
551
        sd_pipe.set_progress_bar_config(disable=None)
552
553

        prompt = "A painting of a squirrel eating a burger"
554
555
556
557
558
559
560
561
562
        generator = torch.Generator(device=device).manual_seed(0)
        output = sd_pipe(
            [prompt],
            generator=generator,
            guidance_scale=6.0,
            num_inference_steps=2,
            output_type="np",
            init_image=init_image,
        )
563

564
565
566
567
568
569
570
571
572
573
574
575
        image = output.images

        generator = torch.Generator(device=device).manual_seed(0)
        image_from_tuple = sd_pipe(
            [prompt],
            generator=generator,
            guidance_scale=6.0,
            num_inference_steps=2,
            output_type="np",
            init_image=init_image,
            return_dict=False,
        )[0]
576
577

        image_slice = image[0, -3:, -3:, -1]
578
        image_from_tuple_slice = image_from_tuple[0, -3:, -3:, -1]
579
580
581
582

        assert image.shape == (1, 32, 32, 3)
        expected_slice = np.array([0.4492, 0.3865, 0.4222, 0.5854, 0.5139, 0.4379, 0.4193, 0.48, 0.4218])
        assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-2
583
        assert np.abs(image_from_tuple_slice.flatten() - expected_slice).max() < 1e-2
584

585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
    def test_stable_diffusion_img2img_k_lms(self):
        device = "cpu"  # ensure determinism for the device-dependent torch.Generator
        unet = self.dummy_cond_unet
        scheduler = LMSDiscreteScheduler(beta_start=0.00085, beta_end=0.012, beta_schedule="scaled_linear")

        vae = self.dummy_vae
        bert = self.dummy_text_encoder
        tokenizer = CLIPTokenizer.from_pretrained("hf-internal-testing/tiny-random-clip")

        init_image = self.dummy_image.to(device)

        # make sure here that pndm scheduler skips prk
        sd_pipe = StableDiffusionImg2ImgPipeline(
            unet=unet,
            scheduler=scheduler,
            vae=vae,
            text_encoder=bert,
            tokenizer=tokenizer,
            safety_checker=self.dummy_safety_checker,
            feature_extractor=self.dummy_extractor,
        )
        sd_pipe = sd_pipe.to(device)
        sd_pipe.set_progress_bar_config(disable=None)

        prompt = "A painting of a squirrel eating a burger"
        generator = torch.Generator(device=device).manual_seed(0)
        output = sd_pipe(
            [prompt],
            generator=generator,
            guidance_scale=6.0,
            num_inference_steps=2,
            output_type="np",
            init_image=init_image,
        )
619
        image = output.images
620

621
622
623
624
625
626
627
628
629
630
631
        generator = torch.Generator(device=device).manual_seed(0)
        output = sd_pipe(
            [prompt],
            generator=generator,
            guidance_scale=6.0,
            num_inference_steps=2,
            output_type="np",
            init_image=init_image,
            return_dict=False,
        )
        image_from_tuple = output[0]
632
633

        image_slice = image[0, -3:, -3:, -1]
634
        image_from_tuple_slice = image_from_tuple[0, -3:, -3:, -1]
635
636
637
638

        assert image.shape == (1, 32, 32, 3)
        expected_slice = np.array([0.4367, 0.4986, 0.4372, 0.6706, 0.5665, 0.444, 0.5864, 0.6019, 0.5203])
        assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-2
639
        assert np.abs(image_from_tuple_slice.flatten() - expected_slice).max() < 1e-2
640

641
    def test_stable_diffusion_inpaint(self):
642
        device = "cpu"  # ensure determinism for the device-dependent torch.Generator
643
644
645
646
647
648
        unet = self.dummy_cond_unet
        scheduler = PNDMScheduler(tensor_format="pt", skip_prk_steps=True)
        vae = self.dummy_vae
        bert = self.dummy_text_encoder
        tokenizer = CLIPTokenizer.from_pretrained("hf-internal-testing/tiny-random-clip")

649
        image = self.dummy_image.to(device).permute(0, 2, 3, 1)[0]
650
651
652
653
654
655
656
657
658
659
660
661
662
        init_image = Image.fromarray(np.uint8(image)).convert("RGB")
        mask_image = Image.fromarray(np.uint8(image + 4)).convert("RGB").resize((128, 128))

        # make sure here that pndm scheduler skips prk
        sd_pipe = StableDiffusionInpaintPipeline(
            unet=unet,
            scheduler=scheduler,
            vae=vae,
            text_encoder=bert,
            tokenizer=tokenizer,
            safety_checker=self.dummy_safety_checker,
            feature_extractor=self.dummy_extractor,
        )
663
        sd_pipe = sd_pipe.to(device)
664
        sd_pipe.set_progress_bar_config(disable=None)
665
666

        prompt = "A painting of a squirrel eating a burger"
667
668
669
670
671
672
673
674
675
676
        generator = torch.Generator(device=device).manual_seed(0)
        output = sd_pipe(
            [prompt],
            generator=generator,
            guidance_scale=6.0,
            num_inference_steps=2,
            output_type="np",
            init_image=init_image,
            mask_image=mask_image,
        )
677

678
679
680
681
682
683
684
685
686
687
688
689
690
        image = output.images

        generator = torch.Generator(device=device).manual_seed(0)
        image_from_tuple = sd_pipe(
            [prompt],
            generator=generator,
            guidance_scale=6.0,
            num_inference_steps=2,
            output_type="np",
            init_image=init_image,
            mask_image=mask_image,
            return_dict=False,
        )[0]
691
692

        image_slice = image[0, -3:, -3:, -1]
693
        image_from_tuple_slice = image_from_tuple[0, -3:, -3:, -1]
694
695
696
697

        assert image.shape == (1, 32, 32, 3)
        expected_slice = np.array([0.4731, 0.5346, 0.4531, 0.6251, 0.5446, 0.4057, 0.5527, 0.5896, 0.5153])
        assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-2
698
        assert np.abs(image_from_tuple_slice.flatten() - expected_slice).max() < 1e-2
699
700


701
class PipelineTesterMixin(unittest.TestCase):
702
703
704
705
706
707
    def tearDown(self):
        # clean up the VRAM after each test
        super().tearDown()
        gc.collect()
        torch.cuda.empty_cache()

708
709
710
711
712
713
714
715
716
717
718
719
720
721
    def test_from_pretrained_save_pretrained(self):
        # 1. Load models
        model = UNet2DModel(
            block_out_channels=(32, 64),
            layers_per_block=2,
            sample_size=32,
            in_channels=3,
            out_channels=3,
            down_block_types=("DownBlock2D", "AttnDownBlock2D"),
            up_block_types=("AttnUpBlock2D", "UpBlock2D"),
        )
        schedular = DDPMScheduler(num_train_timesteps=10)

        ddpm = DDPMPipeline(model, schedular)
722
        ddpm.to(torch_device)
723
        ddpm.set_progress_bar_config(disable=None)
724
725
726
727

        with tempfile.TemporaryDirectory() as tmpdirname:
            ddpm.save_pretrained(tmpdirname)
            new_ddpm = DDPMPipeline.from_pretrained(tmpdirname)
728
            new_ddpm.to(torch_device)
729
730
731

        generator = torch.manual_seed(0)

732
        image = ddpm(generator=generator, output_type="numpy").images
733
        generator = generator.manual_seed(0)
734
        new_image = new_ddpm(generator=generator, output_type="numpy").images
735
736
737
738
739
740
741

        assert np.abs(image - new_image).sum() < 1e-5, "Models don't give the same forward pass"

    @slow
    def test_from_pretrained_hub(self):
        model_path = "google/ddpm-cifar10-32"

742
        scheduler = DDPMScheduler(num_train_timesteps=10)
743

744
745
        ddpm = DDPMPipeline.from_pretrained(model_path, scheduler=scheduler)
        ddpm.to(torch_device)
746
        ddpm.set_progress_bar_config(disable=None)
747
748
        ddpm_from_hub = DiffusionPipeline.from_pretrained(model_path, scheduler=scheduler)
        ddpm_from_hub.to(torch_device)
749
        ddpm_from_hub.set_progress_bar_config(disable=None)
750
751
752

        generator = torch.manual_seed(0)

753
        image = ddpm(generator=generator, output_type="numpy").images
754
        generator = generator.manual_seed(0)
755
        new_image = ddpm_from_hub(generator=generator, output_type="numpy").images
756
757
758
759
760
761
762

        assert np.abs(image - new_image).sum() < 1e-5, "Models don't give the same forward pass"

    @slow
    def test_from_pretrained_hub_pass_model(self):
        model_path = "google/ddpm-cifar10-32"

763
764
        scheduler = DDPMScheduler(num_train_timesteps=10)

765
766
        # pass unet into DiffusionPipeline
        unet = UNet2DModel.from_pretrained(model_path)
767
768
        ddpm_from_hub_custom_model = DiffusionPipeline.from_pretrained(model_path, unet=unet, scheduler=scheduler)
        ddpm_from_hub_custom_model.to(torch_device)
769
        ddpm_from_hub_custom_model.set_progress_bar_config(disable=None)
770

771
772
        ddpm_from_hub = DiffusionPipeline.from_pretrained(model_path, scheduler=scheduler)
        ddpm_from_hub.to(torch_device)
773
        ddpm_from_hub_custom_model.set_progress_bar_config(disable=None)
774
775
776

        generator = torch.manual_seed(0)

777
        image = ddpm_from_hub_custom_model(generator=generator, output_type="numpy").images
778
        generator = generator.manual_seed(0)
779
        new_image = ddpm_from_hub(generator=generator, output_type="numpy").images
780
781
782
783
784
785
786
787

        assert np.abs(image - new_image).sum() < 1e-5, "Models don't give the same forward pass"

    @slow
    def test_output_format(self):
        model_path = "google/ddpm-cifar10-32"

        pipe = DDIMPipeline.from_pretrained(model_path)
788
        pipe.to(torch_device)
789
        pipe.set_progress_bar_config(disable=None)
790
791

        generator = torch.manual_seed(0)
792
        images = pipe(generator=generator, output_type="numpy").images
793
794
795
        assert images.shape == (1, 32, 32, 3)
        assert isinstance(images, np.ndarray)

796
        images = pipe(generator=generator, output_type="pil").images
797
798
799
800
801
        assert isinstance(images, list)
        assert len(images) == 1
        assert isinstance(images[0], PIL.Image.Image)

        # use PIL by default
802
        images = pipe(generator=generator).images
803
804
805
806
807
808
809
810
811
812
813
814
        assert isinstance(images, list)
        assert isinstance(images[0], PIL.Image.Image)

    @slow
    def test_ddpm_cifar10(self):
        model_id = "google/ddpm-cifar10-32"

        unet = UNet2DModel.from_pretrained(model_id)
        scheduler = DDPMScheduler.from_config(model_id)
        scheduler = scheduler.set_format("pt")

        ddpm = DDPMPipeline(unet=unet, scheduler=scheduler)
815
        ddpm.to(torch_device)
816
        ddpm.set_progress_bar_config(disable=None)
817
818

        generator = torch.manual_seed(0)
819
        image = ddpm(generator=generator, output_type="numpy").images
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834

        image_slice = image[0, -3:, -3:, -1]

        assert image.shape == (1, 32, 32, 3)
        expected_slice = np.array([0.41995, 0.35885, 0.19385, 0.38475, 0.3382, 0.2647, 0.41545, 0.3582, 0.33845])
        assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-2

    @slow
    def test_ddim_lsun(self):
        model_id = "google/ddpm-ema-bedroom-256"

        unet = UNet2DModel.from_pretrained(model_id)
        scheduler = DDIMScheduler.from_config(model_id)

        ddpm = DDIMPipeline(unet=unet, scheduler=scheduler)
835
        ddpm.to(torch_device)
836
        ddpm.set_progress_bar_config(disable=None)
837
838

        generator = torch.manual_seed(0)
839
        image = ddpm(generator=generator, output_type="numpy").images
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854

        image_slice = image[0, -3:, -3:, -1]

        assert image.shape == (1, 256, 256, 3)
        expected_slice = np.array([0.00605, 0.0201, 0.0344, 0.00235, 0.00185, 0.00025, 0.00215, 0.0, 0.00685])
        assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-2

    @slow
    def test_ddim_cifar10(self):
        model_id = "google/ddpm-cifar10-32"

        unet = UNet2DModel.from_pretrained(model_id)
        scheduler = DDIMScheduler(tensor_format="pt")

        ddim = DDIMPipeline(unet=unet, scheduler=scheduler)
855
        ddim.to(torch_device)
856
        ddim.set_progress_bar_config(disable=None)
857
858

        generator = torch.manual_seed(0)
859
        image = ddim(generator=generator, eta=0.0, output_type="numpy").images
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874

        image_slice = image[0, -3:, -3:, -1]

        assert image.shape == (1, 32, 32, 3)
        expected_slice = np.array([0.17235, 0.16175, 0.16005, 0.16255, 0.1497, 0.1513, 0.15045, 0.1442, 0.1453])
        assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-2

    @slow
    def test_pndm_cifar10(self):
        model_id = "google/ddpm-cifar10-32"

        unet = UNet2DModel.from_pretrained(model_id)
        scheduler = PNDMScheduler(tensor_format="pt")

        pndm = PNDMPipeline(unet=unet, scheduler=scheduler)
875
        pndm.to(torch_device)
876
        pndm.set_progress_bar_config(disable=None)
877
        generator = torch.manual_seed(0)
878
        image = pndm(generator=generator, output_type="numpy").images
879
880
881
882
883
884
885
886
887
888

        image_slice = image[0, -3:, -3:, -1]

        assert image.shape == (1, 32, 32, 3)
        expected_slice = np.array([0.1564, 0.14645, 0.1406, 0.14715, 0.12425, 0.14045, 0.13115, 0.12175, 0.125])
        assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-2

    @slow
    def test_ldm_text2img(self):
        ldm = LDMTextToImagePipeline.from_pretrained("CompVis/ldm-text2im-large-256")
889
        ldm.to(torch_device)
890
        ldm.set_progress_bar_config(disable=None)
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906

        prompt = "A painting of a squirrel eating a burger"
        generator = torch.manual_seed(0)
        image = ldm([prompt], generator=generator, guidance_scale=6.0, num_inference_steps=20, output_type="numpy")[
            "sample"
        ]

        image_slice = image[0, -3:, -3:, -1]

        assert image.shape == (1, 256, 256, 3)
        expected_slice = np.array([0.9256, 0.9340, 0.8933, 0.9361, 0.9113, 0.8727, 0.9122, 0.8745, 0.8099])
        assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-2

    @slow
    def test_ldm_text2img_fast(self):
        ldm = LDMTextToImagePipeline.from_pretrained("CompVis/ldm-text2im-large-256")
907
        ldm.to(torch_device)
908
        ldm.set_progress_bar_config(disable=None)
909
910
911

        prompt = "A painting of a squirrel eating a burger"
        generator = torch.manual_seed(0)
912
        image = ldm(prompt, generator=generator, num_inference_steps=1, output_type="numpy").images
913
914
915
916
917
918
919
920
921
922
923

        image_slice = image[0, -3:, -3:, -1]

        assert image.shape == (1, 256, 256, 3)
        expected_slice = np.array([0.3163, 0.8670, 0.6465, 0.1865, 0.6291, 0.5139, 0.2824, 0.3723, 0.4344])
        assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-2

    @slow
    @unittest.skipIf(torch_device == "cpu", "Stable diffusion is supposed to run on GPU")
    def test_stable_diffusion(self):
        # make sure here that pndm scheduler skips prk
924
925
        sd_pipe = StableDiffusionPipeline.from_pretrained("CompVis/stable-diffusion-v1-1", use_auth_token=True)
        sd_pipe = sd_pipe.to(torch_device)
926
        sd_pipe.set_progress_bar_config(disable=None)
927
928
929
930
931
932
933
934

        prompt = "A painting of a squirrel eating a burger"
        generator = torch.Generator(device=torch_device).manual_seed(0)
        with torch.autocast("cuda"):
            output = sd_pipe(
                [prompt], generator=generator, guidance_scale=6.0, num_inference_steps=20, output_type="np"
            )

935
        image = output.images
936
937
938
939
940
941
942
943
944
945

        image_slice = image[0, -3:, -3:, -1]

        assert image.shape == (1, 512, 512, 3)
        expected_slice = np.array([0.8887, 0.915, 0.91, 0.894, 0.909, 0.912, 0.919, 0.925, 0.883])
        assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-2

    @slow
    @unittest.skipIf(torch_device == "cpu", "Stable diffusion is supposed to run on GPU")
    def test_stable_diffusion_fast_ddim(self):
946
947
        sd_pipe = StableDiffusionPipeline.from_pretrained("CompVis/stable-diffusion-v1-1", use_auth_token=True)
        sd_pipe = sd_pipe.to(torch_device)
948
        sd_pipe.set_progress_bar_config(disable=None)
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963

        scheduler = DDIMScheduler(
            beta_start=0.00085,
            beta_end=0.012,
            beta_schedule="scaled_linear",
            clip_sample=False,
            set_alpha_to_one=False,
        )
        sd_pipe.scheduler = scheduler

        prompt = "A painting of a squirrel eating a burger"
        generator = torch.Generator(device=torch_device).manual_seed(0)

        with torch.autocast("cuda"):
            output = sd_pipe([prompt], generator=generator, num_inference_steps=2, output_type="numpy")
964
        image = output.images
965
966
967
968

        image_slice = image[0, -3:, -3:, -1]

        assert image.shape == (1, 512, 512, 3)
969
        expected_slice = np.array([0.9326, 0.923, 0.951, 0.9365, 0.9214, 0.951, 0.9365, 0.9414, 0.918])
970
971
972
973
974
975
976
977
978
979
        assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-3

    @slow
    def test_score_sde_ve_pipeline(self):
        model_id = "google/ncsnpp-church-256"
        model = UNet2DModel.from_pretrained(model_id)

        scheduler = ScoreSdeVeScheduler.from_config(model_id)

        sde_ve = ScoreSdeVePipeline(unet=model, scheduler=scheduler)
980
        sde_ve.to(torch_device)
981
        sde_ve.set_progress_bar_config(disable=None)
982
983

        torch.manual_seed(0)
984
        image = sde_ve(num_inference_steps=300, output_type="numpy").images
985
986
987
988
989
990
991
992
993
994
995

        image_slice = image[0, -3:, -3:, -1]

        assert image.shape == (1, 256, 256, 3)

        expected_slice = np.array([0.64363, 0.5868, 0.3031, 0.2284, 0.7409, 0.3216, 0.25643, 0.6557, 0.2633])
        assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-2

    @slow
    def test_ldm_uncond(self):
        ldm = LDMPipeline.from_pretrained("CompVis/ldm-celebahq-256")
996
        ldm.to(torch_device)
997
        ldm.set_progress_bar_config(disable=None)
998
999

        generator = torch.manual_seed(0)
1000
        image = ldm(generator=generator, num_inference_steps=5, output_type="numpy").images
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016

        image_slice = image[0, -3:, -3:, -1]

        assert image.shape == (1, 256, 256, 3)
        expected_slice = np.array([0.4399, 0.44975, 0.46825, 0.474, 0.4359, 0.4581, 0.45095, 0.4341, 0.4447])
        assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-2

    @slow
    def test_ddpm_ddim_equality(self):
        model_id = "google/ddpm-cifar10-32"

        unet = UNet2DModel.from_pretrained(model_id)
        ddpm_scheduler = DDPMScheduler(tensor_format="pt")
        ddim_scheduler = DDIMScheduler(tensor_format="pt")

        ddpm = DDPMPipeline(unet=unet, scheduler=ddpm_scheduler)
1017
        ddpm.to(torch_device)
1018
        ddpm.set_progress_bar_config(disable=None)
1019
        ddim = DDIMPipeline(unet=unet, scheduler=ddim_scheduler)
1020
        ddim.to(torch_device)
1021
        ddim.set_progress_bar_config(disable=None)
1022
1023

        generator = torch.manual_seed(0)
1024
        ddpm_image = ddpm(generator=generator, output_type="numpy").images
1025
1026

        generator = torch.manual_seed(0)
1027
        ddim_image = ddim(generator=generator, num_inference_steps=1000, eta=1.0, output_type="numpy").images
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040

        # the values aren't exactly equal, but the images look the same visually
        assert np.abs(ddpm_image - ddim_image).max() < 1e-1

    @unittest.skip("(Anton) The test is failing for large batch sizes, needs investigation")
    def test_ddpm_ddim_equality_batched(self):
        model_id = "google/ddpm-cifar10-32"

        unet = UNet2DModel.from_pretrained(model_id)
        ddpm_scheduler = DDPMScheduler(tensor_format="pt")
        ddim_scheduler = DDIMScheduler(tensor_format="pt")

        ddpm = DDPMPipeline(unet=unet, scheduler=ddpm_scheduler)
1041
        ddpm.to(torch_device)
1042
        ddpm.set_progress_bar_config(disable=None)
1043

1044
        ddim = DDIMPipeline(unet=unet, scheduler=ddim_scheduler)
1045
        ddim.to(torch_device)
1046
        ddim.set_progress_bar_config(disable=None)
1047
1048

        generator = torch.manual_seed(0)
1049
        ddpm_images = ddpm(batch_size=4, generator=generator, output_type="numpy").images
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065

        generator = torch.manual_seed(0)
        ddim_images = ddim(batch_size=4, generator=generator, num_inference_steps=1000, eta=1.0, output_type="numpy")[
            "sample"
        ]

        # the values aren't exactly equal, but the images look the same visually
        assert np.abs(ddpm_images - ddim_images).max() < 1e-1

    @slow
    def test_karras_ve_pipeline(self):
        model_id = "google/ncsnpp-celebahq-256"
        model = UNet2DModel.from_pretrained(model_id)
        scheduler = KarrasVeScheduler(tensor_format="pt")

        pipe = KarrasVePipeline(unet=model, scheduler=scheduler)
1066
        pipe.to(torch_device)
1067
        pipe.set_progress_bar_config(disable=None)
1068
1069

        generator = torch.manual_seed(0)
1070
        image = pipe(num_inference_steps=20, generator=generator, output_type="numpy").images
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081

        image_slice = image[0, -3:, -3:, -1]
        assert image.shape == (1, 256, 256, 3)
        expected_slice = np.array([0.26815, 0.1581, 0.2658, 0.23248, 0.1550, 0.2539, 0.1131, 0.1024, 0.0837])
        assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-2

    @slow
    @unittest.skipIf(torch_device == "cpu", "Stable diffusion is supposed to run on GPU")
    def test_lms_stable_diffusion_pipeline(self):
        model_id = "CompVis/stable-diffusion-v1-1"
        pipe = StableDiffusionPipeline.from_pretrained(model_id, use_auth_token=True).to(torch_device)
1082
        pipe.set_progress_bar_config(disable=None)
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
        scheduler = LMSDiscreteScheduler.from_config(model_id, subfolder="scheduler", use_auth_token=True)
        pipe.scheduler = scheduler

        prompt = "a photograph of an astronaut riding a horse"
        generator = torch.Generator(device=torch_device).manual_seed(0)
        image = pipe([prompt], generator=generator, guidance_scale=7.5, num_inference_steps=10, output_type="numpy")[
            "sample"
        ]

        image_slice = image[0, -3:, -3:, -1]
        assert image.shape == (1, 512, 512, 3)
        expected_slice = np.array([0.9077, 0.9254, 0.9181, 0.9227, 0.9213, 0.9367, 0.9399, 0.9406, 0.9024])
        assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-2
1096
1097
1098

    @slow
    @unittest.skipIf(torch_device == "cpu", "Stable diffusion is supposed to run on GPU")
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
    def test_stable_diffusion_memory_chunking(self):
        torch.cuda.reset_peak_memory_stats()
        model_id = "CompVis/stable-diffusion-v1-4"
        pipe = StableDiffusionPipeline.from_pretrained(
            model_id, revision="fp16", torch_dtype=torch.float16, use_auth_token=True
        ).to(torch_device)
        pipe.set_progress_bar_config(disable=None)

        prompt = "a photograph of an astronaut riding a horse"

        # make attention efficient
        pipe.enable_attention_slicing()
        generator = torch.Generator(device=torch_device).manual_seed(0)
        with torch.autocast(torch_device):
            output_chunked = pipe(
                [prompt], generator=generator, guidance_scale=7.5, num_inference_steps=10, output_type="numpy"
            )
            image_chunked = output_chunked.images

        mem_bytes = torch.cuda.max_memory_allocated()
        torch.cuda.reset_peak_memory_stats()
        # make sure that less than 3.75 GB is allocated
        assert mem_bytes < 3.75 * 10**9

        # disable chunking
        pipe.disable_attention_slicing()
        generator = torch.Generator(device=torch_device).manual_seed(0)
        with torch.autocast(torch_device):
            output = pipe(
                [prompt], generator=generator, guidance_scale=7.5, num_inference_steps=10, output_type="numpy"
            )
            image = output.images

        # make sure that more than 3.75 GB is allocated
        mem_bytes = torch.cuda.max_memory_allocated()
        assert mem_bytes > 3.75 * 10**9
        assert np.abs(image_chunked.flatten() - image.flatten()).max() < 1e-3

    @slow
    @unittest.skipIf(torch_device == "cpu", "Stable diffusion is supposed to run on GPU")
1139
1140
1141
    def test_stable_diffusion_img2img_pipeline(self):
        ds = load_dataset("hf-internal-testing/diffusers-images", split="train")

1142
        init_image = ds[2]["image"].resize((768, 512))
1143
1144
1145
        output_image = ds[0]["image"].resize((768, 512))

        model_id = "CompVis/stable-diffusion-v1-4"
1146
1147
1148
1149
1150
1151
        pipe = StableDiffusionImg2ImgPipeline.from_pretrained(
            model_id,
            revision="fp16",  # fp16 to infer 768x512 images with 16GB of VRAM
            torch_dtype=torch.float16,
            use_auth_token=True,
        )
1152
        pipe.to(torch_device)
1153
        pipe.set_progress_bar_config(disable=None)
1154
1155
1156
1157

        prompt = "A fantasy landscape, trending on artstation"

        generator = torch.Generator(device=torch_device).manual_seed(0)
1158
1159
1160
        with torch.autocast("cuda"):
            output = pipe(prompt=prompt, init_image=init_image, strength=0.75, guidance_scale=7.5, generator=generator)
        image = output.images[0]
1161

1162
1163
        expected_array = np.array(output_image) / 255.0
        sampled_array = np.array(image) / 255.0
1164
1165
1166
1167
1168
1169

        assert sampled_array.shape == (512, 768, 3)
        assert np.max(np.abs(sampled_array - expected_array)) < 1e-4

    @slow
    @unittest.skipIf(torch_device == "cpu", "Stable diffusion is supposed to run on GPU")
1170
    def test_stable_diffusion_img2img_pipeline_k_lms(self):
1171
1172
1173
        ds = load_dataset("hf-internal-testing/diffusers-images", split="train")

        init_image = ds[2]["image"].resize((768, 512))
1174
1175
1176
1177
1178
        output_image = ds[1]["image"].resize((768, 512))

        lms = LMSDiscreteScheduler(beta_start=0.00085, beta_end=0.012, beta_schedule="scaled_linear")

        model_id = "CompVis/stable-diffusion-v1-4"
1179
1180
1181
1182
1183
1184
1185
        pipe = StableDiffusionImg2ImgPipeline.from_pretrained(
            model_id,
            scheduler=lms,
            revision="fp16",  # fp16 to infer 768x512 images with 16GB of VRAM
            torch_dtype=torch.float16,
            use_auth_token=True,
        )
1186
1187
1188
1189
1190
1191
        pipe.to(torch_device)
        pipe.set_progress_bar_config(disable=None)

        prompt = "A fantasy landscape, trending on artstation"

        generator = torch.Generator(device=torch_device).manual_seed(0)
1192
1193
        with torch.autocast("cuda"):
            output = pipe(prompt=prompt, init_image=init_image, strength=0.75, guidance_scale=7.5, generator=generator)
1194
        image = output.images[0]
1195

1196
1197
        expected_array = np.array(output_image) / 255.0
        sampled_array = np.array(image) / 255.0
1198
1199
1200
1201
1202
1203

        assert sampled_array.shape == (512, 768, 3)
        assert np.max(np.abs(sampled_array - expected_array)) < 1e-4

    @slow
    @unittest.skipIf(torch_device == "cpu", "Stable diffusion is supposed to run on GPU")
1204
    def test_stable_diffusion_inpaint_pipeline(self):
1205
1206
1207
1208
1209
        ds = load_dataset("hf-internal-testing/diffusers-images", split="train")

        init_image = ds[3]["image"].resize((768, 512))
        mask_image = ds[4]["image"].resize((768, 512))
        output_image = ds[5]["image"].resize((768, 512))
1210
1211

        model_id = "CompVis/stable-diffusion-v1-4"
1212
1213
1214
1215
1216
1217
        pipe = StableDiffusionInpaintPipeline.from_pretrained(
            model_id,
            revision="fp16",  # fp16 to infer 768x512 images in 16GB of VRAM
            torch_dtype=torch.float16,
            use_auth_token=True,
        )
1218
        pipe.to(torch_device)
1219
        pipe.set_progress_bar_config(disable=None)
1220
1221
1222
1223

        prompt = "A red cat sitting on a parking bench"

        generator = torch.Generator(device=torch_device).manual_seed(0)
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
        with torch.autocast("cuda"):
            output = pipe(
                prompt=prompt,
                init_image=init_image,
                mask_image=mask_image,
                strength=0.75,
                guidance_scale=7.5,
                generator=generator,
            )
        image = output.images[0]
1234

1235
1236
        expected_array = np.array(output_image) / 255.0
        sampled_array = np.array(image) / 255.0
1237
1238
1239

        assert sampled_array.shape == (512, 768, 3)
        assert np.max(np.abs(sampled_array - expected_array)) < 1e-3