test_pipelines.py 46.1 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
# coding=utf-8
# Copyright 2022 HuggingFace Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

16
import gc
17
import random
18
19
20
21
22
23
24
import tempfile
import unittest

import numpy as np
import torch

import PIL
25
from datasets import load_dataset
26
from diffusers import (
27
    AutoencoderKL,
28
29
30
31
32
33
34
35
36
37
38
39
40
    DDIMPipeline,
    DDIMScheduler,
    DDPMPipeline,
    DDPMScheduler,
    KarrasVePipeline,
    KarrasVeScheduler,
    LDMPipeline,
    LDMTextToImagePipeline,
    LMSDiscreteScheduler,
    PNDMPipeline,
    PNDMScheduler,
    ScoreSdeVePipeline,
    ScoreSdeVeScheduler,
41
42
    StableDiffusionImg2ImgPipeline,
    StableDiffusionInpaintPipeline,
43
    StableDiffusionPipeline,
44
    UNet2DConditionModel,
45
    UNet2DModel,
46
    VQModel,
47
48
)
from diffusers.pipeline_utils import DiffusionPipeline
49
50
51
from diffusers.testing_utils import floats_tensor, slow, torch_device
from PIL import Image
from transformers import CLIPTextConfig, CLIPTextModel, CLIPTokenizer
52
53
54
55
56


torch.backends.cuda.matmul.allow_tf32 = False


hysts's avatar
hysts committed
57
58
59
60
61
62
63
64
65
66
67
68
69
def test_progress_bar(capsys):
    model = UNet2DModel(
        block_out_channels=(32, 64),
        layers_per_block=2,
        sample_size=32,
        in_channels=3,
        out_channels=3,
        down_block_types=("DownBlock2D", "AttnDownBlock2D"),
        up_block_types=("AttnUpBlock2D", "UpBlock2D"),
    )
    scheduler = DDPMScheduler(num_train_timesteps=10)

    ddpm = DDPMPipeline(model, scheduler).to(torch_device)
70
    ddpm(output_type="numpy").images
hysts's avatar
hysts committed
71
72
73
74
    captured = capsys.readouterr()
    assert "10/10" in captured.err, "Progress bar has to be displayed"

    ddpm.set_progress_bar_config(disable=True)
75
    ddpm(output_type="numpy").images
hysts's avatar
hysts committed
76
77
78
79
    captured = capsys.readouterr()
    assert captured.err == "", "Progress bar should be disabled"


80
class PipelineFastTests(unittest.TestCase):
81
82
83
84
85
86
    def tearDown(self):
        # clean up the VRAM after each test
        super().tearDown()
        gc.collect()
        torch.cuda.empty_cache()

87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
    @property
    def dummy_image(self):
        batch_size = 1
        num_channels = 3
        sizes = (32, 32)

        image = floats_tensor((batch_size, num_channels) + sizes, rng=random.Random(0)).to(torch_device)
        return image

    @property
    def dummy_uncond_unet(self):
        torch.manual_seed(0)
        model = UNet2DModel(
            block_out_channels=(32, 64),
            layers_per_block=2,
            sample_size=32,
            in_channels=3,
            out_channels=3,
            down_block_types=("DownBlock2D", "AttnDownBlock2D"),
            up_block_types=("AttnUpBlock2D", "UpBlock2D"),
        )
        return model

    @property
    def dummy_cond_unet(self):
        torch.manual_seed(0)
        model = UNet2DConditionModel(
            block_out_channels=(32, 64),
            layers_per_block=2,
            sample_size=32,
            in_channels=4,
            out_channels=4,
            down_block_types=("DownBlock2D", "CrossAttnDownBlock2D"),
            up_block_types=("CrossAttnUpBlock2D", "UpBlock2D"),
            cross_attention_dim=32,
        )
        return model

    @property
    def dummy_vq_model(self):
        torch.manual_seed(0)
        model = VQModel(
            block_out_channels=[32, 64],
            in_channels=3,
            out_channels=3,
            down_block_types=["DownEncoderBlock2D", "DownEncoderBlock2D"],
            up_block_types=["UpDecoderBlock2D", "UpDecoderBlock2D"],
            latent_channels=3,
        )
        return model

    @property
    def dummy_vae(self):
        torch.manual_seed(0)
        model = AutoencoderKL(
            block_out_channels=[32, 64],
            in_channels=3,
            out_channels=3,
            down_block_types=["DownEncoderBlock2D", "DownEncoderBlock2D"],
            up_block_types=["UpDecoderBlock2D", "UpDecoderBlock2D"],
            latent_channels=4,
        )
        return model

    @property
    def dummy_text_encoder(self):
        torch.manual_seed(0)
        config = CLIPTextConfig(
            bos_token_id=0,
            eos_token_id=2,
            hidden_size=32,
            intermediate_size=37,
            layer_norm_eps=1e-05,
            num_attention_heads=4,
            num_hidden_layers=5,
            pad_token_id=1,
            vocab_size=1000,
        )
        return CLIPTextModel(config)

    @property
    def dummy_safety_checker(self):
        def check(images, *args, **kwargs):
            return images, False

        return check

    @property
    def dummy_extractor(self):
        def extract(*args, **kwargs):
            class Out:
                def __init__(self):
                    self.pixel_values = torch.ones([0])

                def to(self, device):
                    self.pixel_values.to(device)
                    return self

            return Out()

        return extract

    def test_ddim(self):
        unet = self.dummy_uncond_unet
        scheduler = DDIMScheduler(tensor_format="pt")

        ddpm = DDIMPipeline(unet=unet, scheduler=scheduler)
        ddpm.to(torch_device)
195
        ddpm.set_progress_bar_config(disable=None)
196
197

        generator = torch.manual_seed(0)
198
199
200
201
        image = ddpm(generator=generator, num_inference_steps=2, output_type="numpy").images

        generator = torch.manual_seed(0)
        image_from_tuple = ddpm(generator=generator, num_inference_steps=2, output_type="numpy", return_dict=False)[0]
202
203

        image_slice = image[0, -3:, -3:, -1]
204
        image_from_tuple_slice = image_from_tuple[0, -3:, -3:, -1]
205
206
207
208
209
210

        assert image.shape == (1, 32, 32, 3)
        expected_slice = np.array(
            [1.000e00, 5.717e-01, 4.717e-01, 1.000e00, 0.000e00, 1.000e00, 3.000e-04, 0.000e00, 9.000e-04]
        )
        assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-2
211
        assert np.abs(image_from_tuple_slice.flatten() - expected_slice).max() < 1e-2
212
213
214
215
216
217
218

    def test_pndm_cifar10(self):
        unet = self.dummy_uncond_unet
        scheduler = PNDMScheduler(tensor_format="pt")

        pndm = PNDMPipeline(unet=unet, scheduler=scheduler)
        pndm.to(torch_device)
219
        pndm.set_progress_bar_config(disable=None)
220
221
222
223

        generator = torch.manual_seed(0)
        image = pndm(generator=generator, num_inference_steps=20, output_type="numpy").images

224
        generator = torch.manual_seed(0)
225
        image_from_tuple = pndm(generator=generator, num_inference_steps=20, output_type="numpy", return_dict=False)[0]
226
227

        image_slice = image[0, -3:, -3:, -1]
228
        image_from_tuple_slice = image_from_tuple[0, -3:, -3:, -1]
229
230
231
232

        assert image.shape == (1, 32, 32, 3)
        expected_slice = np.array([1.0, 1.0, 0.0, 1.0, 0.0, 1.0, 0.0, 0.0, 0.0])
        assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-2
233
        assert np.abs(image_from_tuple_slice.flatten() - expected_slice).max() < 1e-2
234
235
236
237
238
239
240
241
242
243

    def test_ldm_text2img(self):
        unet = self.dummy_cond_unet
        scheduler = DDIMScheduler(tensor_format="pt")
        vae = self.dummy_vae
        bert = self.dummy_text_encoder
        tokenizer = CLIPTokenizer.from_pretrained("hf-internal-testing/tiny-random-clip")

        ldm = LDMTextToImagePipeline(vqvae=vae, bert=bert, tokenizer=tokenizer, unet=unet, scheduler=scheduler)
        ldm.to(torch_device)
244
        ldm.set_progress_bar_config(disable=None)
245
246
247
248
249
250
251

        prompt = "A painting of a squirrel eating a burger"
        generator = torch.manual_seed(0)
        image = ldm([prompt], generator=generator, guidance_scale=6.0, num_inference_steps=2, output_type="numpy")[
            "sample"
        ]

252
253
254
255
256
257
258
259
260
261
        generator = torch.manual_seed(0)
        image_from_tuple = ldm(
            [prompt],
            generator=generator,
            guidance_scale=6.0,
            num_inference_steps=2,
            output_type="numpy",
            return_dict=False,
        )[0]

262
        image_slice = image[0, -3:, -3:, -1]
263
        image_from_tuple_slice = image_from_tuple[0, -3:, -3:, -1]
264
265
266
267

        assert image.shape == (1, 64, 64, 3)
        expected_slice = np.array([0.5074, 0.5026, 0.4998, 0.4056, 0.3523, 0.4649, 0.5289, 0.5299, 0.4897])
        assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-2
268
        assert np.abs(image_from_tuple_slice.flatten() - expected_slice).max() < 1e-2
269
270

    def test_stable_diffusion_ddim(self):
271
        device = "cpu"  # ensure determinism for the device-dependent torch.Generator
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
        unet = self.dummy_cond_unet
        scheduler = DDIMScheduler(
            beta_start=0.00085,
            beta_end=0.012,
            beta_schedule="scaled_linear",
            clip_sample=False,
            set_alpha_to_one=False,
        )

        vae = self.dummy_vae
        bert = self.dummy_text_encoder
        tokenizer = CLIPTokenizer.from_pretrained("hf-internal-testing/tiny-random-clip")

        # make sure here that pndm scheduler skips prk
        sd_pipe = StableDiffusionPipeline(
            unet=unet,
            scheduler=scheduler,
            vae=vae,
            text_encoder=bert,
            tokenizer=tokenizer,
            safety_checker=self.dummy_safety_checker,
            feature_extractor=self.dummy_extractor,
        )
295
        sd_pipe = sd_pipe.to(device)
296
        sd_pipe.set_progress_bar_config(disable=None)
297
298

        prompt = "A painting of a squirrel eating a burger"
299

300
301
        generator = torch.Generator(device=device).manual_seed(0)
        output = sd_pipe([prompt], generator=generator, guidance_scale=6.0, num_inference_steps=2, output_type="np")
302
        image = output.images
303

304
305
306
307
308
309
310
311
312
        generator = torch.Generator(device=device).manual_seed(0)
        image_from_tuple = sd_pipe(
            [prompt],
            generator=generator,
            guidance_scale=6.0,
            num_inference_steps=2,
            output_type="np",
            return_dict=False,
        )[0]
313
314

        image_slice = image[0, -3:, -3:, -1]
315
        image_from_tuple_slice = image_from_tuple[0, -3:, -3:, -1]
316
317
318
319

        assert image.shape == (1, 128, 128, 3)
        expected_slice = np.array([0.5112, 0.4692, 0.4715, 0.5206, 0.4894, 0.5114, 0.5096, 0.4932, 0.4755])
        assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-2
320
        assert np.abs(image_from_tuple_slice.flatten() - expected_slice).max() < 1e-2
321
322

    def test_stable_diffusion_pndm(self):
323
        device = "cpu"  # ensure determinism for the device-dependent torch.Generator
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
        unet = self.dummy_cond_unet
        scheduler = PNDMScheduler(tensor_format="pt", skip_prk_steps=True)
        vae = self.dummy_vae
        bert = self.dummy_text_encoder
        tokenizer = CLIPTokenizer.from_pretrained("hf-internal-testing/tiny-random-clip")

        # make sure here that pndm scheduler skips prk
        sd_pipe = StableDiffusionPipeline(
            unet=unet,
            scheduler=scheduler,
            vae=vae,
            text_encoder=bert,
            tokenizer=tokenizer,
            safety_checker=self.dummy_safety_checker,
            feature_extractor=self.dummy_extractor,
        )
340
        sd_pipe = sd_pipe.to(device)
341
        sd_pipe.set_progress_bar_config(disable=None)
342
343

        prompt = "A painting of a squirrel eating a burger"
344
345
        generator = torch.Generator(device=device).manual_seed(0)
        output = sd_pipe([prompt], generator=generator, guidance_scale=6.0, num_inference_steps=2, output_type="np")
346

347
348
349
350
351
352
353
354
355
356
357
        image = output.images

        generator = torch.Generator(device=device).manual_seed(0)
        image_from_tuple = sd_pipe(
            [prompt],
            generator=generator,
            guidance_scale=6.0,
            num_inference_steps=2,
            output_type="np",
            return_dict=False,
        )[0]
358
359

        image_slice = image[0, -3:, -3:, -1]
360
        image_from_tuple_slice = image_from_tuple[0, -3:, -3:, -1]
361
362
363
364

        assert image.shape == (1, 128, 128, 3)
        expected_slice = np.array([0.4937, 0.4649, 0.4716, 0.5145, 0.4889, 0.513, 0.513, 0.4905, 0.4738])
        assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-2
365
        assert np.abs(image_from_tuple_slice.flatten() - expected_slice).max() < 1e-2
366
367

    def test_stable_diffusion_k_lms(self):
368
        device = "cpu"  # ensure determinism for the device-dependent torch.Generator
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
        unet = self.dummy_cond_unet
        scheduler = LMSDiscreteScheduler(beta_start=0.00085, beta_end=0.012, beta_schedule="scaled_linear")
        vae = self.dummy_vae
        bert = self.dummy_text_encoder
        tokenizer = CLIPTokenizer.from_pretrained("hf-internal-testing/tiny-random-clip")

        # make sure here that pndm scheduler skips prk
        sd_pipe = StableDiffusionPipeline(
            unet=unet,
            scheduler=scheduler,
            vae=vae,
            text_encoder=bert,
            tokenizer=tokenizer,
            safety_checker=self.dummy_safety_checker,
            feature_extractor=self.dummy_extractor,
        )
385
        sd_pipe = sd_pipe.to(device)
386
        sd_pipe.set_progress_bar_config(disable=None)
387
388

        prompt = "A painting of a squirrel eating a burger"
389
390
        generator = torch.Generator(device=device).manual_seed(0)
        output = sd_pipe([prompt], generator=generator, guidance_scale=6.0, num_inference_steps=2, output_type="np")
391

392
393
394
395
396
397
398
399
400
401
402
        image = output.images

        generator = torch.Generator(device=device).manual_seed(0)
        image_from_tuple = sd_pipe(
            [prompt],
            generator=generator,
            guidance_scale=6.0,
            num_inference_steps=2,
            output_type="np",
            return_dict=False,
        )[0]
403
404

        image_slice = image[0, -3:, -3:, -1]
405
        image_from_tuple_slice = image_from_tuple[0, -3:, -3:, -1]
406
407
408
409

        assert image.shape == (1, 128, 128, 3)
        expected_slice = np.array([0.5067, 0.4689, 0.4614, 0.5233, 0.4903, 0.5112, 0.524, 0.5069, 0.4785])
        assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-2
410
        assert np.abs(image_from_tuple_slice.flatten() - expected_slice).max() < 1e-2
411

412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
    def test_stable_diffusion_attention_chunk(self):
        device = "cpu"  # ensure determinism for the device-dependent torch.Generator
        unet = self.dummy_cond_unet
        scheduler = LMSDiscreteScheduler(beta_start=0.00085, beta_end=0.012, beta_schedule="scaled_linear")
        vae = self.dummy_vae
        bert = self.dummy_text_encoder
        tokenizer = CLIPTokenizer.from_pretrained("hf-internal-testing/tiny-random-clip")

        # make sure here that pndm scheduler skips prk
        sd_pipe = StableDiffusionPipeline(
            unet=unet,
            scheduler=scheduler,
            vae=vae,
            text_encoder=bert,
            tokenizer=tokenizer,
            safety_checker=self.dummy_safety_checker,
            feature_extractor=self.dummy_extractor,
        )
        sd_pipe = sd_pipe.to(device)
        sd_pipe.set_progress_bar_config(disable=None)

        prompt = "A painting of a squirrel eating a burger"
        generator = torch.Generator(device=device).manual_seed(0)
        output_1 = sd_pipe([prompt], generator=generator, guidance_scale=6.0, num_inference_steps=2, output_type="np")

        # make sure chunking the attention yields the same result
        sd_pipe.enable_attention_slicing(slice_size=1)
        generator = torch.Generator(device=device).manual_seed(0)
        output_2 = sd_pipe([prompt], generator=generator, guidance_scale=6.0, num_inference_steps=2, output_type="np")

        assert np.abs(output_2.images.flatten() - output_1.images.flatten()).max() < 1e-4

444
445
446
447
448
449
    def test_score_sde_ve_pipeline(self):
        unet = self.dummy_uncond_unet
        scheduler = ScoreSdeVeScheduler(tensor_format="pt")

        sde_ve = ScoreSdeVePipeline(unet=unet, scheduler=scheduler)
        sde_ve.to(torch_device)
450
        sde_ve.set_progress_bar_config(disable=None)
451
452

        torch.manual_seed(0)
453
454
455
456
        image = sde_ve(num_inference_steps=2, output_type="numpy").images

        torch.manual_seed(0)
        image_from_tuple = sde_ve(num_inference_steps=2, output_type="numpy", return_dict=False)[0]
457
458

        image_slice = image[0, -3:, -3:, -1]
459
        image_from_tuple_slice = image_from_tuple[0, -3:, -3:, -1]
460
461
462
463
464

        assert image.shape == (1, 32, 32, 3)

        expected_slice = np.array([0.0, 1.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0])
        assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-2
465
        assert np.abs(image_from_tuple_slice.flatten() - expected_slice).max() < 1e-2
466
467
468
469
470
471
472
473

    def test_ldm_uncond(self):
        unet = self.dummy_uncond_unet
        scheduler = DDIMScheduler(tensor_format="pt")
        vae = self.dummy_vq_model

        ldm = LDMPipeline(unet=unet, vqvae=vae, scheduler=scheduler)
        ldm.to(torch_device)
474
        ldm.set_progress_bar_config(disable=None)
475
476

        generator = torch.manual_seed(0)
477
478
479
480
        image = ldm(generator=generator, num_inference_steps=2, output_type="numpy").images

        generator = torch.manual_seed(0)
        image_from_tuple = ldm(generator=generator, num_inference_steps=2, output_type="numpy", return_dict=False)[0]
481
482

        image_slice = image[0, -3:, -3:, -1]
483
        image_from_tuple_slice = image_from_tuple[0, -3:, -3:, -1]
484
485
486
487

        assert image.shape == (1, 64, 64, 3)
        expected_slice = np.array([0.8512, 0.818, 0.6411, 0.6808, 0.4465, 0.5618, 0.46, 0.6231, 0.5172])
        assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-2
488
        assert np.abs(image_from_tuple_slice.flatten() - expected_slice).max() < 1e-2
489
490
491
492
493
494
495

    def test_karras_ve_pipeline(self):
        unet = self.dummy_uncond_unet
        scheduler = KarrasVeScheduler(tensor_format="pt")

        pipe = KarrasVePipeline(unet=unet, scheduler=scheduler)
        pipe.to(torch_device)
496
        pipe.set_progress_bar_config(disable=None)
497
498

        generator = torch.manual_seed(0)
499
500
501
502
        image = pipe(num_inference_steps=2, generator=generator, output_type="numpy").images

        generator = torch.manual_seed(0)
        image_from_tuple = pipe(num_inference_steps=2, generator=generator, output_type="numpy", return_dict=False)[0]
503
504

        image_slice = image[0, -3:, -3:, -1]
505
506
        image_from_tuple_slice = image_from_tuple[0, -3:, -3:, -1]

507
508
509
        assert image.shape == (1, 32, 32, 3)
        expected_slice = np.array([0.0, 1.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0])
        assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-2
510
        assert np.abs(image_from_tuple_slice.flatten() - expected_slice).max() < 1e-2
511
512

    def test_stable_diffusion_img2img(self):
513
        device = "cpu"  # ensure determinism for the device-dependent torch.Generator
514
515
516
517
518
519
        unet = self.dummy_cond_unet
        scheduler = PNDMScheduler(tensor_format="pt", skip_prk_steps=True)
        vae = self.dummy_vae
        bert = self.dummy_text_encoder
        tokenizer = CLIPTokenizer.from_pretrained("hf-internal-testing/tiny-random-clip")

520
        init_image = self.dummy_image.to(device)
521
522
523
524
525
526
527
528
529
530
531

        # make sure here that pndm scheduler skips prk
        sd_pipe = StableDiffusionImg2ImgPipeline(
            unet=unet,
            scheduler=scheduler,
            vae=vae,
            text_encoder=bert,
            tokenizer=tokenizer,
            safety_checker=self.dummy_safety_checker,
            feature_extractor=self.dummy_extractor,
        )
532
        sd_pipe = sd_pipe.to(device)
533
        sd_pipe.set_progress_bar_config(disable=None)
534
535

        prompt = "A painting of a squirrel eating a burger"
536
537
538
539
540
541
542
543
544
        generator = torch.Generator(device=device).manual_seed(0)
        output = sd_pipe(
            [prompt],
            generator=generator,
            guidance_scale=6.0,
            num_inference_steps=2,
            output_type="np",
            init_image=init_image,
        )
545

546
547
548
549
550
551
552
553
554
555
556
557
        image = output.images

        generator = torch.Generator(device=device).manual_seed(0)
        image_from_tuple = sd_pipe(
            [prompt],
            generator=generator,
            guidance_scale=6.0,
            num_inference_steps=2,
            output_type="np",
            init_image=init_image,
            return_dict=False,
        )[0]
558
559

        image_slice = image[0, -3:, -3:, -1]
560
        image_from_tuple_slice = image_from_tuple[0, -3:, -3:, -1]
561
562
563
564

        assert image.shape == (1, 32, 32, 3)
        expected_slice = np.array([0.4492, 0.3865, 0.4222, 0.5854, 0.5139, 0.4379, 0.4193, 0.48, 0.4218])
        assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-2
565
        assert np.abs(image_from_tuple_slice.flatten() - expected_slice).max() < 1e-2
566

567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
    def test_stable_diffusion_img2img_k_lms(self):
        device = "cpu"  # ensure determinism for the device-dependent torch.Generator
        unet = self.dummy_cond_unet
        scheduler = LMSDiscreteScheduler(beta_start=0.00085, beta_end=0.012, beta_schedule="scaled_linear")

        vae = self.dummy_vae
        bert = self.dummy_text_encoder
        tokenizer = CLIPTokenizer.from_pretrained("hf-internal-testing/tiny-random-clip")

        init_image = self.dummy_image.to(device)

        # make sure here that pndm scheduler skips prk
        sd_pipe = StableDiffusionImg2ImgPipeline(
            unet=unet,
            scheduler=scheduler,
            vae=vae,
            text_encoder=bert,
            tokenizer=tokenizer,
            safety_checker=self.dummy_safety_checker,
            feature_extractor=self.dummy_extractor,
        )
        sd_pipe = sd_pipe.to(device)
        sd_pipe.set_progress_bar_config(disable=None)

        prompt = "A painting of a squirrel eating a burger"
        generator = torch.Generator(device=device).manual_seed(0)
        output = sd_pipe(
            [prompt],
            generator=generator,
            guidance_scale=6.0,
            num_inference_steps=2,
            output_type="np",
            init_image=init_image,
        )
601
        image = output.images
602

603
604
605
606
607
608
609
610
611
612
613
        generator = torch.Generator(device=device).manual_seed(0)
        output = sd_pipe(
            [prompt],
            generator=generator,
            guidance_scale=6.0,
            num_inference_steps=2,
            output_type="np",
            init_image=init_image,
            return_dict=False,
        )
        image_from_tuple = output[0]
614
615

        image_slice = image[0, -3:, -3:, -1]
616
        image_from_tuple_slice = image_from_tuple[0, -3:, -3:, -1]
617
618
619
620

        assert image.shape == (1, 32, 32, 3)
        expected_slice = np.array([0.4367, 0.4986, 0.4372, 0.6706, 0.5665, 0.444, 0.5864, 0.6019, 0.5203])
        assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-2
621
        assert np.abs(image_from_tuple_slice.flatten() - expected_slice).max() < 1e-2
622

623
    def test_stable_diffusion_inpaint(self):
624
        device = "cpu"  # ensure determinism for the device-dependent torch.Generator
625
626
627
628
629
630
        unet = self.dummy_cond_unet
        scheduler = PNDMScheduler(tensor_format="pt", skip_prk_steps=True)
        vae = self.dummy_vae
        bert = self.dummy_text_encoder
        tokenizer = CLIPTokenizer.from_pretrained("hf-internal-testing/tiny-random-clip")

631
        image = self.dummy_image.to(device).permute(0, 2, 3, 1)[0]
632
633
634
635
636
637
638
639
640
641
642
643
644
        init_image = Image.fromarray(np.uint8(image)).convert("RGB")
        mask_image = Image.fromarray(np.uint8(image + 4)).convert("RGB").resize((128, 128))

        # make sure here that pndm scheduler skips prk
        sd_pipe = StableDiffusionInpaintPipeline(
            unet=unet,
            scheduler=scheduler,
            vae=vae,
            text_encoder=bert,
            tokenizer=tokenizer,
            safety_checker=self.dummy_safety_checker,
            feature_extractor=self.dummy_extractor,
        )
645
        sd_pipe = sd_pipe.to(device)
646
        sd_pipe.set_progress_bar_config(disable=None)
647
648

        prompt = "A painting of a squirrel eating a burger"
649
650
651
652
653
654
655
656
657
658
        generator = torch.Generator(device=device).manual_seed(0)
        output = sd_pipe(
            [prompt],
            generator=generator,
            guidance_scale=6.0,
            num_inference_steps=2,
            output_type="np",
            init_image=init_image,
            mask_image=mask_image,
        )
659

660
661
662
663
664
665
666
667
668
669
670
671
672
        image = output.images

        generator = torch.Generator(device=device).manual_seed(0)
        image_from_tuple = sd_pipe(
            [prompt],
            generator=generator,
            guidance_scale=6.0,
            num_inference_steps=2,
            output_type="np",
            init_image=init_image,
            mask_image=mask_image,
            return_dict=False,
        )[0]
673
674

        image_slice = image[0, -3:, -3:, -1]
675
        image_from_tuple_slice = image_from_tuple[0, -3:, -3:, -1]
676
677
678
679

        assert image.shape == (1, 32, 32, 3)
        expected_slice = np.array([0.4731, 0.5346, 0.4531, 0.6251, 0.5446, 0.4057, 0.5527, 0.5896, 0.5153])
        assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-2
680
        assert np.abs(image_from_tuple_slice.flatten() - expected_slice).max() < 1e-2
681
682


683
class PipelineTesterMixin(unittest.TestCase):
684
685
686
687
688
689
    def tearDown(self):
        # clean up the VRAM after each test
        super().tearDown()
        gc.collect()
        torch.cuda.empty_cache()

690
691
692
693
694
695
696
697
698
699
700
701
702
703
    def test_from_pretrained_save_pretrained(self):
        # 1. Load models
        model = UNet2DModel(
            block_out_channels=(32, 64),
            layers_per_block=2,
            sample_size=32,
            in_channels=3,
            out_channels=3,
            down_block_types=("DownBlock2D", "AttnDownBlock2D"),
            up_block_types=("AttnUpBlock2D", "UpBlock2D"),
        )
        schedular = DDPMScheduler(num_train_timesteps=10)

        ddpm = DDPMPipeline(model, schedular)
704
        ddpm.to(torch_device)
705
        ddpm.set_progress_bar_config(disable=None)
706
707
708
709

        with tempfile.TemporaryDirectory() as tmpdirname:
            ddpm.save_pretrained(tmpdirname)
            new_ddpm = DDPMPipeline.from_pretrained(tmpdirname)
710
            new_ddpm.to(torch_device)
711
712
713

        generator = torch.manual_seed(0)

714
        image = ddpm(generator=generator, output_type="numpy").images
715
        generator = generator.manual_seed(0)
716
        new_image = new_ddpm(generator=generator, output_type="numpy").images
717
718
719
720
721
722
723

        assert np.abs(image - new_image).sum() < 1e-5, "Models don't give the same forward pass"

    @slow
    def test_from_pretrained_hub(self):
        model_path = "google/ddpm-cifar10-32"

724
        scheduler = DDPMScheduler(num_train_timesteps=10)
725

726
727
        ddpm = DDPMPipeline.from_pretrained(model_path, scheduler=scheduler)
        ddpm.to(torch_device)
728
        ddpm.set_progress_bar_config(disable=None)
729
730
        ddpm_from_hub = DiffusionPipeline.from_pretrained(model_path, scheduler=scheduler)
        ddpm_from_hub.to(torch_device)
731
        ddpm_from_hub.set_progress_bar_config(disable=None)
732
733
734

        generator = torch.manual_seed(0)

735
        image = ddpm(generator=generator, output_type="numpy").images
736
        generator = generator.manual_seed(0)
737
        new_image = ddpm_from_hub(generator=generator, output_type="numpy").images
738
739
740
741
742
743
744

        assert np.abs(image - new_image).sum() < 1e-5, "Models don't give the same forward pass"

    @slow
    def test_from_pretrained_hub_pass_model(self):
        model_path = "google/ddpm-cifar10-32"

745
746
        scheduler = DDPMScheduler(num_train_timesteps=10)

747
748
        # pass unet into DiffusionPipeline
        unet = UNet2DModel.from_pretrained(model_path)
749
750
        ddpm_from_hub_custom_model = DiffusionPipeline.from_pretrained(model_path, unet=unet, scheduler=scheduler)
        ddpm_from_hub_custom_model.to(torch_device)
751
        ddpm_from_hub_custom_model.set_progress_bar_config(disable=None)
752

753
754
        ddpm_from_hub = DiffusionPipeline.from_pretrained(model_path, scheduler=scheduler)
        ddpm_from_hub.to(torch_device)
755
        ddpm_from_hub_custom_model.set_progress_bar_config(disable=None)
756
757
758

        generator = torch.manual_seed(0)

759
        image = ddpm_from_hub_custom_model(generator=generator, output_type="numpy").images
760
        generator = generator.manual_seed(0)
761
        new_image = ddpm_from_hub(generator=generator, output_type="numpy").images
762
763
764
765
766
767
768
769

        assert np.abs(image - new_image).sum() < 1e-5, "Models don't give the same forward pass"

    @slow
    def test_output_format(self):
        model_path = "google/ddpm-cifar10-32"

        pipe = DDIMPipeline.from_pretrained(model_path)
770
        pipe.to(torch_device)
771
        pipe.set_progress_bar_config(disable=None)
772
773

        generator = torch.manual_seed(0)
774
        images = pipe(generator=generator, output_type="numpy").images
775
776
777
        assert images.shape == (1, 32, 32, 3)
        assert isinstance(images, np.ndarray)

778
        images = pipe(generator=generator, output_type="pil").images
779
780
781
782
783
        assert isinstance(images, list)
        assert len(images) == 1
        assert isinstance(images[0], PIL.Image.Image)

        # use PIL by default
784
        images = pipe(generator=generator).images
785
786
787
788
789
790
791
792
793
794
795
796
        assert isinstance(images, list)
        assert isinstance(images[0], PIL.Image.Image)

    @slow
    def test_ddpm_cifar10(self):
        model_id = "google/ddpm-cifar10-32"

        unet = UNet2DModel.from_pretrained(model_id)
        scheduler = DDPMScheduler.from_config(model_id)
        scheduler = scheduler.set_format("pt")

        ddpm = DDPMPipeline(unet=unet, scheduler=scheduler)
797
        ddpm.to(torch_device)
798
        ddpm.set_progress_bar_config(disable=None)
799
800

        generator = torch.manual_seed(0)
801
        image = ddpm(generator=generator, output_type="numpy").images
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816

        image_slice = image[0, -3:, -3:, -1]

        assert image.shape == (1, 32, 32, 3)
        expected_slice = np.array([0.41995, 0.35885, 0.19385, 0.38475, 0.3382, 0.2647, 0.41545, 0.3582, 0.33845])
        assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-2

    @slow
    def test_ddim_lsun(self):
        model_id = "google/ddpm-ema-bedroom-256"

        unet = UNet2DModel.from_pretrained(model_id)
        scheduler = DDIMScheduler.from_config(model_id)

        ddpm = DDIMPipeline(unet=unet, scheduler=scheduler)
817
        ddpm.to(torch_device)
818
        ddpm.set_progress_bar_config(disable=None)
819
820

        generator = torch.manual_seed(0)
821
        image = ddpm(generator=generator, output_type="numpy").images
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836

        image_slice = image[0, -3:, -3:, -1]

        assert image.shape == (1, 256, 256, 3)
        expected_slice = np.array([0.00605, 0.0201, 0.0344, 0.00235, 0.00185, 0.00025, 0.00215, 0.0, 0.00685])
        assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-2

    @slow
    def test_ddim_cifar10(self):
        model_id = "google/ddpm-cifar10-32"

        unet = UNet2DModel.from_pretrained(model_id)
        scheduler = DDIMScheduler(tensor_format="pt")

        ddim = DDIMPipeline(unet=unet, scheduler=scheduler)
837
        ddim.to(torch_device)
838
        ddim.set_progress_bar_config(disable=None)
839
840

        generator = torch.manual_seed(0)
841
        image = ddim(generator=generator, eta=0.0, output_type="numpy").images
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856

        image_slice = image[0, -3:, -3:, -1]

        assert image.shape == (1, 32, 32, 3)
        expected_slice = np.array([0.17235, 0.16175, 0.16005, 0.16255, 0.1497, 0.1513, 0.15045, 0.1442, 0.1453])
        assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-2

    @slow
    def test_pndm_cifar10(self):
        model_id = "google/ddpm-cifar10-32"

        unet = UNet2DModel.from_pretrained(model_id)
        scheduler = PNDMScheduler(tensor_format="pt")

        pndm = PNDMPipeline(unet=unet, scheduler=scheduler)
857
        pndm.to(torch_device)
858
        pndm.set_progress_bar_config(disable=None)
859
        generator = torch.manual_seed(0)
860
        image = pndm(generator=generator, output_type="numpy").images
861
862
863
864
865
866
867
868
869
870

        image_slice = image[0, -3:, -3:, -1]

        assert image.shape == (1, 32, 32, 3)
        expected_slice = np.array([0.1564, 0.14645, 0.1406, 0.14715, 0.12425, 0.14045, 0.13115, 0.12175, 0.125])
        assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-2

    @slow
    def test_ldm_text2img(self):
        ldm = LDMTextToImagePipeline.from_pretrained("CompVis/ldm-text2im-large-256")
871
        ldm.to(torch_device)
872
        ldm.set_progress_bar_config(disable=None)
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888

        prompt = "A painting of a squirrel eating a burger"
        generator = torch.manual_seed(0)
        image = ldm([prompt], generator=generator, guidance_scale=6.0, num_inference_steps=20, output_type="numpy")[
            "sample"
        ]

        image_slice = image[0, -3:, -3:, -1]

        assert image.shape == (1, 256, 256, 3)
        expected_slice = np.array([0.9256, 0.9340, 0.8933, 0.9361, 0.9113, 0.8727, 0.9122, 0.8745, 0.8099])
        assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-2

    @slow
    def test_ldm_text2img_fast(self):
        ldm = LDMTextToImagePipeline.from_pretrained("CompVis/ldm-text2im-large-256")
889
        ldm.to(torch_device)
890
        ldm.set_progress_bar_config(disable=None)
891
892
893

        prompt = "A painting of a squirrel eating a burger"
        generator = torch.manual_seed(0)
894
        image = ldm(prompt, generator=generator, num_inference_steps=1, output_type="numpy").images
895
896
897
898
899
900
901
902
903
904
905

        image_slice = image[0, -3:, -3:, -1]

        assert image.shape == (1, 256, 256, 3)
        expected_slice = np.array([0.3163, 0.8670, 0.6465, 0.1865, 0.6291, 0.5139, 0.2824, 0.3723, 0.4344])
        assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-2

    @slow
    @unittest.skipIf(torch_device == "cpu", "Stable diffusion is supposed to run on GPU")
    def test_stable_diffusion(self):
        # make sure here that pndm scheduler skips prk
906
907
        sd_pipe = StableDiffusionPipeline.from_pretrained("CompVis/stable-diffusion-v1-1", use_auth_token=True)
        sd_pipe = sd_pipe.to(torch_device)
908
        sd_pipe.set_progress_bar_config(disable=None)
909
910
911
912
913
914
915
916

        prompt = "A painting of a squirrel eating a burger"
        generator = torch.Generator(device=torch_device).manual_seed(0)
        with torch.autocast("cuda"):
            output = sd_pipe(
                [prompt], generator=generator, guidance_scale=6.0, num_inference_steps=20, output_type="np"
            )

917
        image = output.images
918
919
920
921
922
923
924
925
926
927

        image_slice = image[0, -3:, -3:, -1]

        assert image.shape == (1, 512, 512, 3)
        expected_slice = np.array([0.8887, 0.915, 0.91, 0.894, 0.909, 0.912, 0.919, 0.925, 0.883])
        assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-2

    @slow
    @unittest.skipIf(torch_device == "cpu", "Stable diffusion is supposed to run on GPU")
    def test_stable_diffusion_fast_ddim(self):
928
929
        sd_pipe = StableDiffusionPipeline.from_pretrained("CompVis/stable-diffusion-v1-1", use_auth_token=True)
        sd_pipe = sd_pipe.to(torch_device)
930
        sd_pipe.set_progress_bar_config(disable=None)
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945

        scheduler = DDIMScheduler(
            beta_start=0.00085,
            beta_end=0.012,
            beta_schedule="scaled_linear",
            clip_sample=False,
            set_alpha_to_one=False,
        )
        sd_pipe.scheduler = scheduler

        prompt = "A painting of a squirrel eating a burger"
        generator = torch.Generator(device=torch_device).manual_seed(0)

        with torch.autocast("cuda"):
            output = sd_pipe([prompt], generator=generator, num_inference_steps=2, output_type="numpy")
946
        image = output.images
947
948
949
950

        image_slice = image[0, -3:, -3:, -1]

        assert image.shape == (1, 512, 512, 3)
951
        expected_slice = np.array([0.9326, 0.923, 0.951, 0.9365, 0.9214, 0.951, 0.9365, 0.9414, 0.918])
952
953
954
955
956
957
958
959
960
961
        assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-3

    @slow
    def test_score_sde_ve_pipeline(self):
        model_id = "google/ncsnpp-church-256"
        model = UNet2DModel.from_pretrained(model_id)

        scheduler = ScoreSdeVeScheduler.from_config(model_id)

        sde_ve = ScoreSdeVePipeline(unet=model, scheduler=scheduler)
962
        sde_ve.to(torch_device)
963
        sde_ve.set_progress_bar_config(disable=None)
964
965

        torch.manual_seed(0)
966
        image = sde_ve(num_inference_steps=300, output_type="numpy").images
967
968
969
970
971
972
973
974
975
976
977

        image_slice = image[0, -3:, -3:, -1]

        assert image.shape == (1, 256, 256, 3)

        expected_slice = np.array([0.64363, 0.5868, 0.3031, 0.2284, 0.7409, 0.3216, 0.25643, 0.6557, 0.2633])
        assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-2

    @slow
    def test_ldm_uncond(self):
        ldm = LDMPipeline.from_pretrained("CompVis/ldm-celebahq-256")
978
        ldm.to(torch_device)
979
        ldm.set_progress_bar_config(disable=None)
980
981

        generator = torch.manual_seed(0)
982
        image = ldm(generator=generator, num_inference_steps=5, output_type="numpy").images
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998

        image_slice = image[0, -3:, -3:, -1]

        assert image.shape == (1, 256, 256, 3)
        expected_slice = np.array([0.4399, 0.44975, 0.46825, 0.474, 0.4359, 0.4581, 0.45095, 0.4341, 0.4447])
        assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-2

    @slow
    def test_ddpm_ddim_equality(self):
        model_id = "google/ddpm-cifar10-32"

        unet = UNet2DModel.from_pretrained(model_id)
        ddpm_scheduler = DDPMScheduler(tensor_format="pt")
        ddim_scheduler = DDIMScheduler(tensor_format="pt")

        ddpm = DDPMPipeline(unet=unet, scheduler=ddpm_scheduler)
999
        ddpm.to(torch_device)
1000
        ddpm.set_progress_bar_config(disable=None)
1001
        ddim = DDIMPipeline(unet=unet, scheduler=ddim_scheduler)
1002
        ddim.to(torch_device)
1003
        ddim.set_progress_bar_config(disable=None)
1004
1005

        generator = torch.manual_seed(0)
1006
        ddpm_image = ddpm(generator=generator, output_type="numpy").images
1007
1008

        generator = torch.manual_seed(0)
1009
        ddim_image = ddim(generator=generator, num_inference_steps=1000, eta=1.0, output_type="numpy").images
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022

        # the values aren't exactly equal, but the images look the same visually
        assert np.abs(ddpm_image - ddim_image).max() < 1e-1

    @unittest.skip("(Anton) The test is failing for large batch sizes, needs investigation")
    def test_ddpm_ddim_equality_batched(self):
        model_id = "google/ddpm-cifar10-32"

        unet = UNet2DModel.from_pretrained(model_id)
        ddpm_scheduler = DDPMScheduler(tensor_format="pt")
        ddim_scheduler = DDIMScheduler(tensor_format="pt")

        ddpm = DDPMPipeline(unet=unet, scheduler=ddpm_scheduler)
1023
        ddpm.to(torch_device)
1024
        ddpm.set_progress_bar_config(disable=None)
1025

1026
        ddim = DDIMPipeline(unet=unet, scheduler=ddim_scheduler)
1027
        ddim.to(torch_device)
1028
        ddim.set_progress_bar_config(disable=None)
1029
1030

        generator = torch.manual_seed(0)
1031
        ddpm_images = ddpm(batch_size=4, generator=generator, output_type="numpy").images
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047

        generator = torch.manual_seed(0)
        ddim_images = ddim(batch_size=4, generator=generator, num_inference_steps=1000, eta=1.0, output_type="numpy")[
            "sample"
        ]

        # the values aren't exactly equal, but the images look the same visually
        assert np.abs(ddpm_images - ddim_images).max() < 1e-1

    @slow
    def test_karras_ve_pipeline(self):
        model_id = "google/ncsnpp-celebahq-256"
        model = UNet2DModel.from_pretrained(model_id)
        scheduler = KarrasVeScheduler(tensor_format="pt")

        pipe = KarrasVePipeline(unet=model, scheduler=scheduler)
1048
        pipe.to(torch_device)
1049
        pipe.set_progress_bar_config(disable=None)
1050
1051

        generator = torch.manual_seed(0)
1052
        image = pipe(num_inference_steps=20, generator=generator, output_type="numpy").images
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063

        image_slice = image[0, -3:, -3:, -1]
        assert image.shape == (1, 256, 256, 3)
        expected_slice = np.array([0.26815, 0.1581, 0.2658, 0.23248, 0.1550, 0.2539, 0.1131, 0.1024, 0.0837])
        assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-2

    @slow
    @unittest.skipIf(torch_device == "cpu", "Stable diffusion is supposed to run on GPU")
    def test_lms_stable_diffusion_pipeline(self):
        model_id = "CompVis/stable-diffusion-v1-1"
        pipe = StableDiffusionPipeline.from_pretrained(model_id, use_auth_token=True).to(torch_device)
1064
        pipe.set_progress_bar_config(disable=None)
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
        scheduler = LMSDiscreteScheduler.from_config(model_id, subfolder="scheduler", use_auth_token=True)
        pipe.scheduler = scheduler

        prompt = "a photograph of an astronaut riding a horse"
        generator = torch.Generator(device=torch_device).manual_seed(0)
        image = pipe([prompt], generator=generator, guidance_scale=7.5, num_inference_steps=10, output_type="numpy")[
            "sample"
        ]

        image_slice = image[0, -3:, -3:, -1]
        assert image.shape == (1, 512, 512, 3)
        expected_slice = np.array([0.9077, 0.9254, 0.9181, 0.9227, 0.9213, 0.9367, 0.9399, 0.9406, 0.9024])
        assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-2
1078
1079
1080

    @slow
    @unittest.skipIf(torch_device == "cpu", "Stable diffusion is supposed to run on GPU")
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
    def test_stable_diffusion_memory_chunking(self):
        torch.cuda.reset_peak_memory_stats()
        model_id = "CompVis/stable-diffusion-v1-4"
        pipe = StableDiffusionPipeline.from_pretrained(
            model_id, revision="fp16", torch_dtype=torch.float16, use_auth_token=True
        ).to(torch_device)
        pipe.set_progress_bar_config(disable=None)

        prompt = "a photograph of an astronaut riding a horse"

        # make attention efficient
        pipe.enable_attention_slicing()
        generator = torch.Generator(device=torch_device).manual_seed(0)
        with torch.autocast(torch_device):
            output_chunked = pipe(
                [prompt], generator=generator, guidance_scale=7.5, num_inference_steps=10, output_type="numpy"
            )
            image_chunked = output_chunked.images

        mem_bytes = torch.cuda.max_memory_allocated()
        torch.cuda.reset_peak_memory_stats()
        # make sure that less than 3.75 GB is allocated
        assert mem_bytes < 3.75 * 10**9

        # disable chunking
        pipe.disable_attention_slicing()
        generator = torch.Generator(device=torch_device).manual_seed(0)
        with torch.autocast(torch_device):
            output = pipe(
                [prompt], generator=generator, guidance_scale=7.5, num_inference_steps=10, output_type="numpy"
            )
            image = output.images

        # make sure that more than 3.75 GB is allocated
        mem_bytes = torch.cuda.max_memory_allocated()
        assert mem_bytes > 3.75 * 10**9
        assert np.abs(image_chunked.flatten() - image.flatten()).max() < 1e-3

    @slow
    @unittest.skipIf(torch_device == "cpu", "Stable diffusion is supposed to run on GPU")
1121
1122
1123
    def test_stable_diffusion_img2img_pipeline(self):
        ds = load_dataset("hf-internal-testing/diffusers-images", split="train")

1124
        init_image = ds[2]["image"].resize((768, 512))
1125
1126
1127
        output_image = ds[0]["image"].resize((768, 512))

        model_id = "CompVis/stable-diffusion-v1-4"
1128
1129
1130
1131
1132
1133
        pipe = StableDiffusionImg2ImgPipeline.from_pretrained(
            model_id,
            revision="fp16",  # fp16 to infer 768x512 images with 16GB of VRAM
            torch_dtype=torch.float16,
            use_auth_token=True,
        )
1134
        pipe.to(torch_device)
1135
        pipe.set_progress_bar_config(disable=None)
1136
1137
1138
1139

        prompt = "A fantasy landscape, trending on artstation"

        generator = torch.Generator(device=torch_device).manual_seed(0)
1140
1141
1142
        with torch.autocast("cuda"):
            output = pipe(prompt=prompt, init_image=init_image, strength=0.75, guidance_scale=7.5, generator=generator)
        image = output.images[0]
1143

1144
1145
        expected_array = np.array(output_image) / 255.0
        sampled_array = np.array(image) / 255.0
1146
1147
1148
1149
1150
1151

        assert sampled_array.shape == (512, 768, 3)
        assert np.max(np.abs(sampled_array - expected_array)) < 1e-4

    @slow
    @unittest.skipIf(torch_device == "cpu", "Stable diffusion is supposed to run on GPU")
1152
    def test_stable_diffusion_img2img_pipeline_k_lms(self):
1153
1154
1155
        ds = load_dataset("hf-internal-testing/diffusers-images", split="train")

        init_image = ds[2]["image"].resize((768, 512))
1156
1157
1158
1159
1160
        output_image = ds[1]["image"].resize((768, 512))

        lms = LMSDiscreteScheduler(beta_start=0.00085, beta_end=0.012, beta_schedule="scaled_linear")

        model_id = "CompVis/stable-diffusion-v1-4"
1161
1162
1163
1164
1165
1166
1167
        pipe = StableDiffusionImg2ImgPipeline.from_pretrained(
            model_id,
            scheduler=lms,
            revision="fp16",  # fp16 to infer 768x512 images with 16GB of VRAM
            torch_dtype=torch.float16,
            use_auth_token=True,
        )
1168
1169
1170
1171
1172
1173
        pipe.to(torch_device)
        pipe.set_progress_bar_config(disable=None)

        prompt = "A fantasy landscape, trending on artstation"

        generator = torch.Generator(device=torch_device).manual_seed(0)
1174
1175
        with torch.autocast("cuda"):
            output = pipe(prompt=prompt, init_image=init_image, strength=0.75, guidance_scale=7.5, generator=generator)
1176
        image = output.images[0]
1177

1178
1179
        expected_array = np.array(output_image) / 255.0
        sampled_array = np.array(image) / 255.0
1180
1181
1182
1183
1184
1185

        assert sampled_array.shape == (512, 768, 3)
        assert np.max(np.abs(sampled_array - expected_array)) < 1e-4

    @slow
    @unittest.skipIf(torch_device == "cpu", "Stable diffusion is supposed to run on GPU")
1186
    def test_stable_diffusion_inpaint_pipeline(self):
1187
1188
1189
1190
1191
        ds = load_dataset("hf-internal-testing/diffusers-images", split="train")

        init_image = ds[3]["image"].resize((768, 512))
        mask_image = ds[4]["image"].resize((768, 512))
        output_image = ds[5]["image"].resize((768, 512))
1192
1193

        model_id = "CompVis/stable-diffusion-v1-4"
1194
1195
1196
1197
1198
1199
        pipe = StableDiffusionInpaintPipeline.from_pretrained(
            model_id,
            revision="fp16",  # fp16 to infer 768x512 images in 16GB of VRAM
            torch_dtype=torch.float16,
            use_auth_token=True,
        )
1200
        pipe.to(torch_device)
1201
        pipe.set_progress_bar_config(disable=None)
1202
1203
1204
1205

        prompt = "A red cat sitting on a parking bench"

        generator = torch.Generator(device=torch_device).manual_seed(0)
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
        with torch.autocast("cuda"):
            output = pipe(
                prompt=prompt,
                init_image=init_image,
                mask_image=mask_image,
                strength=0.75,
                guidance_scale=7.5,
                generator=generator,
            )
        image = output.images[0]
1216

1217
1218
        expected_array = np.array(output_image) / 255.0
        sampled_array = np.array(image) / 255.0
1219
1220
1221

        assert sampled_array.shape == (512, 768, 3)
        assert np.max(np.abs(sampled_array - expected_array)) < 1e-3